db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-12-04 10:44:46 +02:00
Code Issues Releases Activity

Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into testgres1

Browse Source
This commit is contained in:
sfalkon 2017-11-07 11:42:54 +03:00
commit 72aa1592c3
2 changed files with 74 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
tests/helpers/ptrack_helpers.py
View File

@ -223,15 +223,47 @@ class ProbackupTest(object):
return os.path.join(node.base_dir, 'data', return os.path.join(node.base_dir, 'data',
node.execute("postgres", "select pg_relation_filepath('{0}')".format(fork_name))[0][0]) node.execute("postgres", "select pg_relation_filepath('{0}')".format(fork_name))[0][0])
def get_md5_per_page_for_fork(self, file, size): def get_md5_per_page_for_fork(self, file, size_in_pages):
file = os.open(file, os.O_RDONLY) pages_per_segment = {}
offset = 0
md5_per_page = {} md5_per_page = {}
for page in range(size): nsegments = size_in_pages/131072
md5_per_page[page] = hashlib.md5(os.read(file, 8192)).hexdigest() if size_in_pages%131072 != 0:
nsegments = nsegments + 1
#print("Size: {0}".format(size_in_pages))
#print("Number of segments: {0}".format(nsegments))
size = size_in_pages
for segment_number in range(nsegments):
if size-131072 > 0:
pages_per_segment[segment_number] = 131072
else:
pages_per_segment[segment_number] = size
size = size-131072
#print(pages_per_segment)
for segment_number in range(nsegments):
offset = 0
# print("Segno: {0}".format(segment_number))
# print("Number of pages: {0}".format(pages_per_segment[segment_number]))
if segment_number == 0:
file_desc = os.open(file, os.O_RDONLY)
start_page = 0
end_page = pages_per_segment[segment_number]
else:
file_desc = os.open(file+".{0}".format(segment_number), os.O_RDONLY)
start_page = max(md5_per_page)+1
end_page = end_page + pages_per_segment[segment_number]
# print('Start Page: {0}'.format(start_page))
for page in range(start_page, end_page):
md5_per_page[page] = hashlib.md5(os.read(file_desc, 8192)).hexdigest()
offset += 8192 offset += 8192
os.lseek(file, offset, 0) os.lseek(file_desc, offset, 0)
os.close(file) # print('End Page: {0}'.format(max(md5_per_page)))
os.close(file_desc)
#print("Total Size: {0}".format(len(md5_per_page)))
return md5_per_page return md5_per_page
def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]):
@ -278,7 +310,7 @@ class ProbackupTest(object):
if self.verbose: if self.verbose:
print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format( print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict) # print(idx_dict)
success = False success = False
continue continue
if PageNum not in idx_dict['new_pages']: if PageNum not in idx_dict['new_pages']:
@ -297,7 +329,8 @@ class ProbackupTest(object):
if self.verbose: if self.verbose:
print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format( print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict) print("\n Old checksumm: {0}\n New checksumm: {1}".format(idx_dict['old_pages'][PageNum], idx_dict['new_pages'][PageNum]))
#print(idx_dict)
if PageNum == 0 and idx_dict['type'] == 'spgist': if PageNum == 0 and idx_dict['type'] == 'spgist':
if self.verbose: if self.verbose:
print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0') print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0')
@ -309,9 +342,9 @@ class ProbackupTest(object):
if self.verbose: if self.verbose:
print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format( print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict) #print(idx_dict)
self.assertEqual(success, True, 'Ptrack of index {0} does not correspond to state of its pages.\n Gory Details: \n{1}'.format( #self.assertEqual(success, True, 'Ptrack does not correspond to state of its pages.\n Gory Details: \n{0}'.format(
idx_dict['type'], idx_dict)) # idx_dict['type'], idx_dict))
def check_ptrack_recovery(self, idx_dict): def check_ptrack_recovery(self, idx_dict):
size = idx_dict['size'] size = idx_dict['size']
@ -602,7 +635,7 @@ class ProbackupTest(object):
def switch_wal_segment(self, node): def switch_wal_segment(self, node):
""" Execute pg_switch_wal/xlog() in given node""" """ Execute pg_switch_wal/xlog() in given node"""
if version_to_num(node.safe_psql("postgres", "show server_version")) >= version_to_num('10.0'): if testgres.version_to_num(node.safe_psql("postgres", "show server_version")) >= testgres.version_to_num('10.0'):
node.safe_psql("postgres", "select pg_switch_wal()") node.safe_psql("postgres", "select pg_switch_wal()")
else: else:
node.safe_psql("postgres", "select pg_switch_xlog()") node.safe_psql("postgres", "select pg_switch_xlog()")

32
tests/ptrack.py
View File

@ -721,11 +721,12 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
initdb_params=['--data-checksums'], initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', pg_options={'wal_level': 'replica', 'max_wal_senders': '2',
'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '1GB', 'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '1GB',
'maintenance_work_mem': '1GB', 'autovacuum': 'off'} 'maintenance_work_mem': '1GB', 'autovacuum': 'off', 'full_page_writes': 'off'}
) )
self.init_pb(backup_dir) self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node) self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start() node.start()
self.create_tblspace_in_node(node, 'somedata') self.create_tblspace_in_node(node, 'somedata')
@ -733,17 +734,38 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
# CREATE TABLE # CREATE TABLE
node.pgbench_init(scale=100, options=['--tablespace=somedata']) node.pgbench_init(scale=100, options=['--tablespace=somedata'])
# FULL BACKUP # FULL BACKUP
self.backup_node(backup_dir, 'node', node, options=["--stream"]) #self.backup_node(backup_dir, 'node', node, options=["--stream"])
self.backup_node(backup_dir, 'node', node)
# PTRACK STUFF
idx_ptrack = {'type': 'heap'}
idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts')
idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts')
idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack['path'], idx_ptrack['old_size'])
pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum']) pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum'])
pgbench.wait() pgbench.wait()
#node.safe_psql("postgres", "update pgbench_accounts set bid = bid +1")
node.safe_psql("postgres", "checkpoint")
idx_ptrack['new_size'] = self.get_fork_size(node, 'pgbench_accounts')
idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork(idx_ptrack['path'], idx_ptrack['new_size'])
idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
node, idx_ptrack['path'], [idx_ptrack['old_size'], idx_ptrack['new_size']])
self.check_ptrack_sanity(idx_ptrack)
## PTRACK STUFF
# GET LOGICAL CONTENT FROM NODE # GET LOGICAL CONTENT FROM NODE
result = node.safe_psql("postgres", "select * from pgbench_accounts") result = node.safe_psql("postgres", "select * from pgbench_accounts")
# FIRTS PTRACK BACKUP # FIRTS PTRACK BACKUP
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"]) #self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
# GET PHYSICAL CONTENT FROM NODE # GET PHYSICAL CONTENT FROM NODE
pgdata = self.pgdata_content(node.data_dir) pgdata = self.pgdata_content(node.data_dir)
#get_md5_per_page_for_fork
# RESTORE NODE # RESTORE NODE
restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname)) restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname))
@ -766,10 +788,10 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts") result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts")
# COMPARE RESTORED FILES # COMPARE RESTORED FILES
self.assertEqual(result, result_new) self.assertEqual(result, result_new, 'data is lost')
if self.paranoia: if self.paranoia:
self.compare_pgdata(pgdata, pgdata_restored) self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself # Clean after yourself
self.del_test_dir(module_name, fname) # self.del_test_dir(module_name, fname)