2017-06-20 12:57:23 +02:00
|
|
|
import os
|
2017-05-03 13:14:48 +02:00
|
|
|
import unittest
|
2017-06-27 07:42:52 +02:00
|
|
|
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
|
2017-05-03 13:14:48 +02:00
|
|
|
|
|
|
|
|
2017-07-12 16:28:28 +02:00
|
|
|
module_name = 'ptrack_vacuum_bits_frozen'
|
|
|
|
|
|
|
|
|
2017-05-03 13:14:48 +02:00
|
|
|
class SimpleTest(ProbackupTest, unittest.TestCase):
|
|
|
|
|
2017-05-22 13:17:43 +02:00
|
|
|
# @unittest.skip("skip")
|
|
|
|
# @unittest.expectedFailure
|
2017-05-03 13:14:48 +02:00
|
|
|
def test_ptrack_vacuum_bits_frozen(self):
|
2017-05-17 11:46:38 +02:00
|
|
|
fname = self.id().split('.')[3]
|
2018-10-16 21:53:59 +02:00
|
|
|
node = self.make_simple_node(
|
|
|
|
base_dir="{0}/{1}/node".format(module_name, fname),
|
2017-05-03 13:14:48 +02:00
|
|
|
set_replication=True,
|
2017-06-20 12:57:23 +02:00
|
|
|
initdb_params=['--data-checksums'],
|
2018-10-16 21:53:59 +02:00
|
|
|
pg_options={
|
|
|
|
'ptrack_enable': 'on',
|
|
|
|
'wal_level': 'replica',
|
|
|
|
'max_wal_senders': '2'})
|
|
|
|
|
2017-07-12 16:28:28 +02:00
|
|
|
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
2017-06-20 12:57:23 +02:00
|
|
|
self.init_pb(backup_dir)
|
|
|
|
self.add_instance(backup_dir, 'node', node)
|
2018-10-18 08:46:02 +02:00
|
|
|
node.slow_start()
|
2017-06-20 12:57:23 +02:00
|
|
|
|
2017-05-03 13:14:48 +02:00
|
|
|
self.create_tblspace_in_node(node, 'somedata')
|
|
|
|
|
|
|
|
# Create table and indexes
|
2017-07-12 16:28:28 +02:00
|
|
|
res = node.safe_psql(
|
|
|
|
"postgres",
|
2018-10-19 13:54:17 +02:00
|
|
|
"create extension bloom; create sequence t_seq; "
|
|
|
|
"create table t_heap tablespace somedata "
|
2018-10-16 21:53:59 +02:00
|
|
|
"as select i as id, md5(i::text) as text, "
|
|
|
|
"md5(repeat(i::text,10))::tsvector as tsvector "
|
|
|
|
"from generate_series(0,2560) i")
|
2017-07-12 16:28:28 +02:00
|
|
|
for i in idx_ptrack:
|
|
|
|
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
|
2018-10-16 21:53:59 +02:00
|
|
|
node.safe_psql(
|
|
|
|
"postgres",
|
|
|
|
"create index {0} on {1} using {2}({3}) "
|
|
|
|
"tablespace somedata".format(
|
|
|
|
i, idx_ptrack[i]['relation'],
|
|
|
|
idx_ptrack[i]['type'],
|
|
|
|
idx_ptrack[i]['column']))
|
2017-07-12 16:28:28 +02:00
|
|
|
|
|
|
|
node.safe_psql('postgres', 'checkpoint')
|
|
|
|
|
2018-10-17 16:04:23 +02:00
|
|
|
self.backup_node(
|
|
|
|
backup_dir, 'node', node, options=['-j10', '--stream'])
|
|
|
|
|
|
|
|
node.safe_psql('postgres', 'vacuum freeze t_heap')
|
|
|
|
node.safe_psql('postgres', 'checkpoint')
|
|
|
|
|
2017-07-12 16:28:28 +02:00
|
|
|
for i in idx_ptrack:
|
|
|
|
# get size of heap and indexes. size calculated in pages
|
|
|
|
idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
|
|
|
|
# get path to heap and index files
|
|
|
|
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
|
|
|
|
# calculate md5sums of pages
|
|
|
|
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
|
|
|
|
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
|
|
|
|
|
2018-10-16 21:53:59 +02:00
|
|
|
# CHECK PTRACK SANITY
|
|
|
|
success = True
|
2017-07-12 16:28:28 +02:00
|
|
|
for i in idx_ptrack:
|
|
|
|
# get new size of heap and indexes. size calculated in pages
|
|
|
|
idx_ptrack[i]['new_size'] = self.get_fork_size(node, i)
|
|
|
|
# update path to heap and index files in case they`ve changed
|
|
|
|
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
|
|
|
|
# calculate new md5sums for pages
|
|
|
|
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
|
|
|
|
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
|
|
|
|
# get ptrack for every idx
|
|
|
|
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
|
2018-10-16 21:53:59 +02:00
|
|
|
node, idx_ptrack[i]['path'],
|
|
|
|
[idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
|
2017-07-12 16:28:28 +02:00
|
|
|
|
|
|
|
# compare pages and check ptrack sanity
|
2018-10-16 21:53:59 +02:00
|
|
|
if not self.check_ptrack_sanity(idx_ptrack[i]):
|
|
|
|
success = False
|
|
|
|
|
|
|
|
self.assertTrue(
|
|
|
|
success, 'Ptrack has failed to register changes in data files'
|
|
|
|
)
|
2017-07-12 16:28:28 +02:00
|
|
|
|
|
|
|
# Clean after yourself
|
|
|
|
self.del_test_dir(module_name, fname)
|
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
# @unittest.skip("skip")
|
2017-07-12 16:28:28 +02:00
|
|
|
def test_ptrack_vacuum_bits_frozen_replica(self):
|
|
|
|
fname = self.id().split('.')[3]
|
2018-10-16 21:53:59 +02:00
|
|
|
master = self.make_simple_node(
|
|
|
|
base_dir="{0}/{1}/master".format(module_name, fname),
|
2017-07-12 16:28:28 +02:00
|
|
|
set_replication=True,
|
|
|
|
initdb_params=['--data-checksums'],
|
2018-10-16 21:53:59 +02:00
|
|
|
pg_options={
|
|
|
|
'ptrack_enable': 'on',
|
|
|
|
'wal_level': 'replica',
|
|
|
|
'max_wal_senders': '2'})
|
|
|
|
|
2017-07-12 16:28:28 +02:00
|
|
|
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
|
|
|
self.init_pb(backup_dir)
|
|
|
|
self.add_instance(backup_dir, 'master', master)
|
2018-10-18 08:46:02 +02:00
|
|
|
master.slow_start()
|
2017-07-12 16:28:28 +02:00
|
|
|
|
|
|
|
self.backup_node(backup_dir, 'master', master, options=['--stream'])
|
|
|
|
|
2018-10-16 21:53:59 +02:00
|
|
|
replica = self.make_simple_node(
|
|
|
|
base_dir="{0}/{1}/replica".format(module_name, fname))
|
2017-09-28 09:32:06 +02:00
|
|
|
replica.cleanup()
|
2017-07-12 16:28:28 +02:00
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
self.restore_node(backup_dir, 'master', replica)
|
2017-07-12 16:28:28 +02:00
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
self.add_instance(backup_dir, 'replica', replica)
|
|
|
|
self.set_replica(master, replica, synchronous=True)
|
|
|
|
self.set_archiving(backup_dir, 'replica', replica, replica=True)
|
2018-10-18 08:46:02 +02:00
|
|
|
replica.slow_start(replica=True)
|
2017-07-12 16:28:28 +02:00
|
|
|
|
|
|
|
# Create table and indexes
|
2017-09-28 09:32:06 +02:00
|
|
|
master.safe_psql(
|
2017-05-03 13:14:48 +02:00
|
|
|
"postgres",
|
2018-10-19 13:54:17 +02:00
|
|
|
"create extension bloom; create sequence t_seq; "
|
|
|
|
"create table t_heap as select i as id, "
|
2018-10-16 21:53:59 +02:00
|
|
|
"md5(i::text) as text, md5(repeat(i::text,10))::tsvector "
|
|
|
|
"as tsvector from generate_series(0,2560) i")
|
2017-05-03 13:14:48 +02:00
|
|
|
for i in idx_ptrack:
|
2017-07-12 16:28:28 +02:00
|
|
|
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
|
2018-10-16 21:53:59 +02:00
|
|
|
master.safe_psql(
|
|
|
|
"postgres",
|
|
|
|
"create index {0} on {1} using {2}({3})".format(
|
|
|
|
i, idx_ptrack[i]['relation'],
|
|
|
|
idx_ptrack[i]['type'],
|
|
|
|
idx_ptrack[i]['column']))
|
2017-05-03 13:14:48 +02:00
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
master.safe_psql('postgres', 'checkpoint')
|
2017-05-03 13:14:48 +02:00
|
|
|
|
2018-10-16 21:53:59 +02:00
|
|
|
# Sync master and replica
|
2018-10-18 08:46:02 +02:00
|
|
|
self.wait_until_replica_catch_with_master(master, replica)
|
2018-10-16 21:53:59 +02:00
|
|
|
replica.safe_psql('postgres', 'checkpoint')
|
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
# Take PTRACK backup to clean every ptrack
|
2018-10-16 21:53:59 +02:00
|
|
|
self.backup_node(
|
|
|
|
backup_dir, 'replica', replica,
|
|
|
|
options=[
|
|
|
|
'-j10',
|
|
|
|
'--master-host=localhost',
|
|
|
|
'--master-db=postgres',
|
|
|
|
'--master-port={0}'.format(master.port)])
|
2017-05-03 13:14:48 +02:00
|
|
|
|
2018-10-17 16:04:23 +02:00
|
|
|
for i in idx_ptrack:
|
|
|
|
# get size of heap and indexes. size calculated in pages
|
|
|
|
idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
|
|
|
|
# get path to heap and index files
|
|
|
|
idx_ptrack[i]['path'] = self.get_fork_path(replica, i)
|
|
|
|
# calculate md5sums of pages
|
|
|
|
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
|
|
|
|
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
|
|
|
|
|
2017-09-28 09:32:06 +02:00
|
|
|
master.safe_psql('postgres', 'vacuum freeze t_heap')
|
|
|
|
master.safe_psql('postgres', 'checkpoint')
|
2017-05-03 13:14:48 +02:00
|
|
|
|
2018-10-16 21:53:59 +02:00
|
|
|
# Sync master and replica
|
2018-10-18 08:46:02 +02:00
|
|
|
self.wait_until_replica_catch_with_master(master, replica)
|
2018-10-16 21:53:59 +02:00
|
|
|
replica.safe_psql('postgres', 'checkpoint')
|
|
|
|
|
|
|
|
# CHECK PTRACK SANITY
|
|
|
|
success = True
|
2017-05-03 13:14:48 +02:00
|
|
|
for i in idx_ptrack:
|
|
|
|
# get new size of heap and indexes. size calculated in pages
|
2017-09-28 09:32:06 +02:00
|
|
|
idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
|
2017-05-03 13:14:48 +02:00
|
|
|
# update path to heap and index files in case they`ve changed
|
2017-09-28 09:32:06 +02:00
|
|
|
idx_ptrack[i]['path'] = self.get_fork_path(replica, i)
|
2017-05-03 13:14:48 +02:00
|
|
|
# calculate new md5sums for pages
|
|
|
|
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
|
2017-05-03 17:05:19 +02:00
|
|
|
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
|
2017-05-03 13:14:48 +02:00
|
|
|
# get ptrack for every idx
|
2017-05-05 15:21:49 +02:00
|
|
|
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
|
2018-10-16 21:53:59 +02:00
|
|
|
replica, idx_ptrack[i]['path'],
|
|
|
|
[idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
|
2017-05-03 13:14:48 +02:00
|
|
|
|
|
|
|
# compare pages and check ptrack sanity
|
2018-10-16 21:53:59 +02:00
|
|
|
if not self.check_ptrack_sanity(idx_ptrack[i]):
|
|
|
|
success = False
|
|
|
|
|
|
|
|
self.assertTrue(
|
|
|
|
success, 'Ptrack has failed to register changes in data files'
|
|
|
|
)
|
2017-05-03 13:14:48 +02:00
|
|
|
|
2017-06-27 07:42:52 +02:00
|
|
|
# Clean after yourself
|
2017-07-12 16:28:28 +02:00
|
|
|
self.del_test_dir(module_name, fname)
|