db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-28 09:33:54 +02:00
Code Issues Releases Activity

Merge branch 'release_2_4' into issue_205

Browse Source
This commit is contained in:
Grigory Smolkin 2020-05-19 21:21:32 +03:00
commit c363de2280
10 changed files with 100 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -1,4 +1,5 @@
[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup)
[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest)
# pg_probackup

10
doc/pgprobackup.xml
View File

@ -1168,12 +1168,12 @@ CREATE EXTENSION ptrack;
</listitem>
<listitem>
<para>
To enable tracking page updates, set <varname>ptrack_map_size</varname>
To enable tracking page updates, set <varname>ptrack.map_size</varname>
parameter to a positive integer and restart the server.
</para>
<para>
For optimal performance, it is recommended to set
<varname>ptrack_map_size</varname> to
<varname>ptrack.map_size</varname> to
<literal><replaceable>N</replaceable> / 1024</literal>, where
<replaceable>N</replaceable> is the size of the
<productname>PostgreSQL</productname> cluster, in MB. If you set this
@ -1181,7 +1181,7 @@ CREATE EXTENSION ptrack;
together, which leads to false-positive results when tracking changed
blocks and increases the incremental backup size as unchanged blocks
can also be copied into the incremental backup.
Setting <varname>ptrack_map_size</varname> to a higher value
Setting <varname>ptrack.map_size</varname> to a higher value
does not affect PTRACK operation. The maximum allowed value is 1024.
</para>
</listitem>
@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup;
<note>
<para>
If you change the <varname>ptrack_map_size</varname> parameter value,
If you change the <varname>ptrack.map_size</varname> parameter value,
the previously created PTRACK map file is cleared,
and tracking newly changed blocks starts from scratch. Thus, you have
to retake a full backup before taking incremental PTRACK backups after
changing <varname>ptrack_map_size</varname>.
changing <varname>ptrack.map_size</varname>.
</para>
</note>

16
src/backup.c
View File

@ -435,10 +435,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
/*
* Build the page map from ptrack information.
*/
if (nodeInfo->ptrack_version_num == 20)
if (nodeInfo->ptrack_version_num >= 20)
make_pagemap_from_ptrack_2(backup_files_list, backup_conn,
nodeInfo->ptrack_schema,
prev_backup_start_lsn);
nodeInfo->ptrack_schema,
nodeInfo->ptrack_version_num,
prev_backup_start_lsn);
else if (nodeInfo->ptrack_version_num == 15 ||
nodeInfo->ptrack_version_num == 16 ||
nodeInfo->ptrack_version_num == 17)
@ -862,15 +863,10 @@ do_backup(time_t start_time, bool no_validate,
#endif
get_ptrack_version(backup_conn, &nodeInfo);
// elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
// elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
if (nodeInfo.ptrack_version_num > 0)
{
if (nodeInfo.ptrack_version_num >= 20)
nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn);
else
nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn);
}
nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num);
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{

17
src/data.c
View File

@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg,
}
}
/* Get page via ptrack interface from PostgreSQL shared buffer.
* We do this in following cases:
* 1. PTRACK backup of 1.x versions
* 2. During backup, regardless of backup mode, of PostgreSQL instance
* with ptrack support we encountered invalid page.
/*
* Get page via ptrack interface from PostgreSQL shared buffer.
* We do this only in the cases of PTRACK 1.x versions backup
*/
if ((backup_mode == BACKUP_MODE_DIFF_PTRACK
if (backup_mode == BACKUP_MODE_DIFF_PTRACK
&& (ptrack_version_num >= 15 && ptrack_version_num < 20))
|| !page_is_valid)
{
int rc = 0;
size_t page_size = 0;
@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg,
memcpy(page, ptrack_page, BLCKSZ);
pg_free(ptrack_page);
/* UPD: It apprears that is possible to get zeroed page or page with invalid header
/*
* UPD: It apprears that is possible to get zeroed page or page with invalid header
* from shared buffer.
* Note, that getting page with wrong checksumm from shared buffer is
* acceptable.
@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg,
from_fullpath, blknum, errormsg);
}
/* We must set checksum here, because it is outdated
/*
* We must set checksum here, because it is outdated
* in the block recieved from shared buffers.
*/
if (checksum_version)

10
src/pg_probackup.h
View File

@ -1009,11 +1009,12 @@ extern void parse_filelist_filenames(parray *files, const char *root);
/* in ptrack.c */
extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn);
extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn,
const char *ptrack_schema, XLogRecPtr lsn);
const char *ptrack_schema,
int ptrack_version_num,
XLogRecPtr lsn);
extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num);
extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo);
extern bool pg_ptrack_enable(PGconn *backup_conn);
extern bool pg_ptrack_enable2(PGconn *backup_conn);
extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num);
extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn);
extern char *pg_ptrack_get_and_clear(Oid tablespace_oid,
Oid db_oid,
@ -1021,7 +1022,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid,
size_t *result_size,
PGconn *backup_conn);
extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo);
extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn);
extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema,
int ptrack_version_num, XLogRecPtr lsn);
/* FIO */
extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn,

81
src/ptrack.c
View File

@ -199,6 +199,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo)
nodeInfo->ptrack_version_num = 17;
else if (strcmp(ptrack_version_str, "2.0") == 0)
nodeInfo->ptrack_version_num = 20;
else if (strcmp(ptrack_version_str, "2.1") == 0)
nodeInfo->ptrack_version_num = 21;
else
elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s",
ptrack_version_str);
@ -210,19 +212,30 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo)
* Check if ptrack is enabled in target instance
*/
bool
pg_ptrack_enable(PGconn *backup_conn)
pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num)
{
PGresult *res_db;
bool result = false;
res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL);
if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0)
if (ptrack_version_num < 20)
{
PQclear(res_db);
return false;
res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL);
result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0;
}
else if (ptrack_version_num == 20)
{
res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL);
result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0;
}
else
{
res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL);
result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 &&
strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0;
}
PQclear(res_db);
return true;
return result;
}
@ -452,7 +465,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo)
{
char query[128];
sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema);
if (nodeInfo->ptrack_version_num == 20)
sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema);
else
sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema);
res = pgut_execute(backup_conn, query, 0, NULL);
}
@ -504,7 +521,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
if (arguments->cancel_conn == NULL)
arguments->cancel_conn = PQgetCancel(arguments->conn);
//elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum);
// elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum);
if (ptrack_version_num < 20)
res = pgut_execute_parallel(arguments->conn,
@ -519,7 +536,11 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
if (!ptrack_schema)
elog(ERROR, "Schema name of ptrack extension is missing");
sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
if (ptrack_version_num == 20)
sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
else
elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()");
// sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
res = pgut_execute_parallel(arguments->conn,
arguments->cancel_conn,
@ -559,30 +580,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
* ----------------------------
*/
/*
* Check if ptrack is enabled in target instance
*/
bool
pg_ptrack_enable2(PGconn *backup_conn)
{
PGresult *res_db;
res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL);
if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0)
{
PQclear(res_db);
return false;
}
PQclear(res_db);
return true;
}
/*
* Fetch a list of changed files with their ptrack maps.
*/
parray *
pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn)
pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema,
int ptrack_version_num, XLogRecPtr lsn)
{
PGresult *res;
char lsn_buf[17 + 1];
@ -597,8 +600,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec
if (!ptrack_schema)
elog(ERROR, "Schema name of ptrack extension is missing");
sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1",
ptrack_schema);
if (ptrack_version_num == 20)
sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1",
ptrack_schema);
else
sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1",
ptrack_schema);
res = pgut_execute(backup_conn, query, 1, (const char **) params);
pfree(params[0]);
@ -640,16 +647,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec
*/
void
make_pagemap_from_ptrack_2(parray *files,
PGconn *backup_conn,
const char *ptrack_schema,
XLogRecPtr lsn)
PGconn *backup_conn,
const char *ptrack_schema,
int ptrack_version_num,
XLogRecPtr lsn)
{
parray *filemaps;
int file_i = 0;
page_map_entry *dummy_map = NULL;
/* Receive all available ptrack bitmaps at once */
filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn);
filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema,
ptrack_version_num, lsn);
if (filemaps != NULL)
parray_qsort(filemaps, pgFileMapComparePath);

3
tests/helpers/ptrack_helpers.py
View File

@ -366,7 +366,8 @@ class ProbackupTest(object):
if ptrack_enable:
if node.major_version > 11:
options['ptrack_map_size'] = '128MB'
options['ptrack.map_size'] = '128'
options['shared_preload_libraries'] = 'ptrack'
else:
options['ptrack_enable'] = 'on'

34
tests/ptrack.py
View File

@ -269,7 +269,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True, initdb_params=['--data-checksums'],
pg_options={
'checkpoint_timeout': '30s'})
'checkpoint_timeout': '30s',
'shared_preload_libraries': 'ptrack'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
@ -336,16 +337,16 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
# DISABLE PTRACK
if node.major_version >= 12:
node.safe_psql('postgres', "alter system set ptrack_map_size to 0")
node.safe_psql('postgres', "alter system set ptrack.map_size to 0")
else:
node.safe_psql('postgres', "alter system set ptrack_enable to off")
node.stop()
node.slow_start()
# ENABLE PTRACK
if node.major_version >= 12:
node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'")
node.safe_psql('postgres', "alter system set ptrack.map_size to '128'")
node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'")
else:
node.safe_psql('postgres', "alter system set ptrack_enable to on")
node.stop()
@ -412,7 +413,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node, backup_type='ptrack',
options=['--stream'])
pgdata = self.pgdata_content(node.data_dir)
# TODO: what's the point in taking pgdata content, then taking
# backup, and the trying to compare those two? Backup issues a
# checkpoint, so it will modify pgdata with close to 100% chance.
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
self.backup_node(
backup_dir, 'node', node, backup_type='ptrack',
@ -426,8 +431,9 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
backup_dir, 'node', node_restored,
node_restored.data_dir, options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node_restored.data_dir, ignore_ptrack=False)
if self.paranoia:
pgdata_restored = self.pgdata_content(
node_restored.data_dir, ignore_ptrack=False)
self.set_auto_conf(
node_restored, {'port': node_restored.port})
@ -435,7 +441,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
node_restored.slow_start()
# Physical comparison
self.compare_pgdata(pgdata, pgdata_restored)
if self.paranoia:
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
@ -630,6 +637,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
node.slow_start()
if node.major_version >= 12:
self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*")
node.safe_psql(
"postgres",
"CREATE EXTENSION ptrack")
@ -3841,7 +3849,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
@unittest.expectedFailure
# @unittest.expectedFailure
def test_ptrack_pg_resetxlog(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(
@ -3946,7 +3954,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
)
except ProbackupException as e:
self.assertIn(
'Insert error message',
'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup',
e.message,
'\n Unexpected Error Message: {0}\n'
' CMD: {1}'.format(repr(e.message), self.cmd))
@ -4051,10 +4059,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
log_content = f.read()
self.assertIn(
'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map),
'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map),
log_content)
self.set_auto_conf(node, {'ptrack_map_size': '0'})
self.set_auto_conf(node, {'ptrack.map_size': '0'})
node.slow_start()
@ -4082,7 +4090,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
node.stop(['-m', 'immediate', '-D', node.data_dir])
self.set_auto_conf(node, {'ptrack_map_size': '32'})
self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'})
node.slow_start()

6
travis/Dockerfile.in
View File

@ -2,7 +2,11 @@ FROM ololobus/postgres-dev:stretch
USER root
RUN apt-get update
RUN apt-get -yq install python python-pip python-virtualenv
RUN apt-get -yq install python python-pip
# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
# RUN python2 get-pip.py
RUN python2 -m pip install virtualenv
# Environment
ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH}

2
travis/run_tests.sh
View File

@ -60,7 +60,7 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install
# Setup python environment
echo "############### Setting up python env:"
virtualenv pyenv
python2 -m virtualenv pyenv
source pyenv/bin/activate
pip install testgres==1.8.2