diff --git a/README.md b/README.md index e908f273..ad098872 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) +[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) # pg_probackup diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 58d42749..a2161f0c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1168,12 +1168,12 @@ CREATE EXTENSION ptrack; - To enable tracking page updates, set ptrack_map_size + To enable tracking page updates, set ptrack.map_size parameter to a positive integer and restart the server. For optimal performance, it is recommended to set - ptrack_map_size to + ptrack.map_size to N / 1024, where N is the size of the PostgreSQL cluster, in MB. If you set this @@ -1181,7 +1181,7 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack_map_size to a higher value + Setting ptrack.map_size to a higher value does not affect PTRACK operation. The maximum allowed value is 1024. @@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - If you change the ptrack_map_size parameter value, + If you change the ptrack.map_size parameter value, the previously created PTRACK map file is cleared, and tracking newly changed blocks starts from scratch. Thus, you have to retake a full backup before taking incremental PTRACK backups after - changing ptrack_map_size. + changing ptrack.map_size. diff --git a/src/backup.c b/src/backup.c index 0d84d82a..b421ab58 100644 --- a/src/backup.c +++ b/src/backup.c @@ -435,10 +435,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num == 20) + if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - prev_backup_start_lsn); + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); else if (nodeInfo->ptrack_version_num == 15 || nodeInfo->ptrack_version_num == 16 || nodeInfo->ptrack_version_num == 17) @@ -875,15 +876,10 @@ do_backup(time_t start_time, bool no_validate, #endif get_ptrack_version(backup_conn, &nodeInfo); -// elog(WARNING, "ptrack_version_num %d", ptrack_version_num); + // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - { - if (nodeInfo.ptrack_version_num >= 20) - nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn); - else - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn); - } + nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { diff --git a/src/data.c b/src/data.c index 1270de4f..83e95c86 100644 --- a/src/data.c +++ b/src/data.c @@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg, } } - /* Get page via ptrack interface from PostgreSQL shared buffer. - * We do this in following cases: - * 1. PTRACK backup of 1.x versions - * 2. During backup, regardless of backup mode, of PostgreSQL instance - * with ptrack support we encountered invalid page. + /* + * Get page via ptrack interface from PostgreSQL shared buffer. + * We do this only in the cases of PTRACK 1.x versions backup */ - if ((backup_mode == BACKUP_MODE_DIFF_PTRACK + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && (ptrack_version_num >= 15 && ptrack_version_num < 20)) - || !page_is_valid) { int rc = 0; size_t page_size = 0; @@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg, memcpy(page, ptrack_page, BLCKSZ); pg_free(ptrack_page); - /* UPD: It apprears that is possible to get zeroed page or page with invalid header + /* + * UPD: It apprears that is possible to get zeroed page or page with invalid header * from shared buffer. * Note, that getting page with wrong checksumm from shared buffer is * acceptable. @@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg, from_fullpath, blknum, errormsg); } - /* We must set checksum here, because it is outdated + /* + * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ if (checksum_version) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc24d3e6..3ce98c4a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1005,11 +1005,12 @@ extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, - const char *ptrack_schema, XLogRecPtr lsn); + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn); extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn); -extern bool pg_ptrack_enable2(PGconn *backup_conn); +extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, @@ -1017,7 +1018,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, size_t *result_size, PGconn *backup_conn); extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn); /* FIO */ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, diff --git a/src/ptrack.c b/src/ptrack.c index d5905b51..3f259113 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -199,6 +199,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = 17; else if (strcmp(ptrack_version_str, "2.0") == 0) nodeInfo->ptrack_version_num = 20; + else if (strcmp(ptrack_version_str, "2.1") == 0) + nodeInfo->ptrack_version_num = 21; else elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", ptrack_version_str); @@ -210,19 +212,30 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn) +pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; + bool result = false; - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + if (ptrack_version_num < 20) { - PQclear(res_db); - return false; + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; } + else if (ptrack_version_num == 20) + { + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + } + else + { + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; + } + PQclear(res_db); - return true; + return result; } @@ -452,7 +465,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) { char query[128]; - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + res = pgut_execute(backup_conn, query, 0, NULL); } @@ -504,7 +521,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (arguments->cancel_conn == NULL) arguments->cancel_conn = PQgetCancel(arguments->conn); - //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); if (ptrack_version_num < 20) res = pgut_execute_parallel(arguments->conn, @@ -519,7 +536,11 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + else + elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); + // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, @@ -559,30 +580,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments, * ---------------------------- */ -/* - * Check if ptrack is enabled in target instance - */ -bool -pg_ptrack_enable2(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - return true; -} - /* * Fetch a list of changed files with their ptrack maps. */ parray * -pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn) +pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn) { PGresult *res; char lsn_buf[17 + 1]; @@ -597,8 +600,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); + else + sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); res = pgut_execute(backup_conn, query, 1, (const char **) params); pfree(params[0]); @@ -640,16 +647,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec */ void make_pagemap_from_ptrack_2(parray *files, - PGconn *backup_conn, - const char *ptrack_schema, - XLogRecPtr lsn) + PGconn *backup_conn, + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn) { parray *filemaps; int file_i = 0; page_map_entry *dummy_map = NULL; /* Receive all available ptrack bitmaps at once */ - filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn); + filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, + ptrack_version_num, lsn); if (filemaps != NULL) parray_qsort(filemaps, pgFileMapComparePath); diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 52a1ffda..eec57a3b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -366,7 +366,8 @@ class ProbackupTest(object): if ptrack_enable: if node.major_version > 11: - options['ptrack_map_size'] = '128MB' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' else: options['ptrack_enable'] = 'on' diff --git a/tests/ptrack.py b/tests/ptrack.py index 309a3c44..7d304d52 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -269,7 +269,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '30s', + 'shared_preload_libraries': 'ptrack'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -336,16 +337,16 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # DISABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to 0") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") else: node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.stop() node.slow_start() # ENABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") else: node.safe_psql('postgres', "alter system set ptrack_enable to on") node.stop() @@ -412,7 +413,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) + # TODO: what's the point in taking pgdata content, then taking + # backup, and the trying to compare those two? Backup issues a + # checkpoint, so it will modify pgdata with close to 100% chance. + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -426,8 +431,9 @@ class PtrackTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node_restored, node_restored.data_dir, options=["-j", "4"]) - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.set_auto_conf( node_restored, {'port': node_restored.port}) @@ -435,7 +441,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node_restored.slow_start() # Physical comparison - self.compare_pgdata(pgdata, pgdata_restored) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -630,6 +637,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.slow_start() if node.major_version >= 12: + self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3841,7 +3849,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - @unittest.expectedFailure + # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): fname = self.id().split('.')[3] node = self.make_simple_node( @@ -3946,7 +3954,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) except ProbackupException as e: self.assertIn( - 'Insert error message', + 'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) @@ -4051,10 +4059,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): log_content = f.read() self.assertIn( - 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) - self.set_auto_conf(node, {'ptrack_map_size': '0'}) + self.set_auto_conf(node, {'ptrack.map_size': '0'}) node.slow_start() @@ -4082,7 +4090,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.stop(['-m', 'immediate', '-D', node.data_dir]) - self.set_auto_conf(node, {'ptrack_map_size': '32'}) + self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) node.slow_start() diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index 68f5ffe2..a3c858ee 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,7 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv +RUN apt-get -yq install python python-pip + +# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +# RUN python2 get-pip.py +RUN python2 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 95cb646c..1bb3a6fd 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -60,7 +60,7 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -virtualenv pyenv +python2 -m virtualenv pyenv source pyenv/bin/activate pip install testgres==1.8.2