mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2024-12-03 09:59:53 +02:00
Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into testgres1
This commit is contained in:
sfalkon
commit
72aa1592c3
@ -223,15 +223,47 @@ class ProbackupTest(object):
|
||||
return os.path.join(node.base_dir, 'data',
|
||||
node.execute("postgres", "select pg_relation_filepath('{0}')".format(fork_name))[0][0])
|
||||
|
||||
def get_md5_per_page_for_fork(self, file, size):
|
||||
file = os.open(file, os.O_RDONLY)
|
||||
offset = 0
|
||||
def get_md5_per_page_for_fork(self, file, size_in_pages):
|
||||
pages_per_segment = {}
|
||||
md5_per_page = {}
|
||||
for page in range(size):
|
||||
md5_per_page[page] = hashlib.md5(os.read(file, 8192)).hexdigest()
|
||||
offset += 8192
|
||||
os.lseek(file, offset, 0)
|
||||
os.close(file)
|
||||
nsegments = size_in_pages/131072
|
||||
if size_in_pages%131072 != 0:
|
||||
nsegments = nsegments + 1
|
||||
#print("Size: {0}".format(size_in_pages))
|
||||
#print("Number of segments: {0}".format(nsegments))
|
||||
|
||||
size = size_in_pages
|
||||
for segment_number in range(nsegments):
|
||||
if size-131072 > 0:
|
||||
pages_per_segment[segment_number] = 131072
|
||||
else:
|
||||
pages_per_segment[segment_number] = size
|
||||
size = size-131072
|
||||
|
||||
#print(pages_per_segment)
|
||||
|
||||
for segment_number in range(nsegments):
|
||||
offset = 0
|
||||
# print("Segno: {0}".format(segment_number))
|
||||
# print("Number of pages: {0}".format(pages_per_segment[segment_number]))
|
||||
if segment_number == 0:
|
||||
file_desc = os.open(file, os.O_RDONLY)
|
||||
start_page = 0
|
||||
end_page = pages_per_segment[segment_number]
|
||||
else:
|
||||
file_desc = os.open(file+".{0}".format(segment_number), os.O_RDONLY)
|
||||
start_page = max(md5_per_page)+1
|
||||
end_page = end_page + pages_per_segment[segment_number]
|
||||
|
||||
# print('Start Page: {0}'.format(start_page))
|
||||
for page in range(start_page, end_page):
|
||||
md5_per_page[page] = hashlib.md5(os.read(file_desc, 8192)).hexdigest()
|
||||
offset += 8192
|
||||
os.lseek(file_desc, offset, 0)
|
||||
# print('End Page: {0}'.format(max(md5_per_page)))
|
||||
os.close(file_desc)
|
||||
|
||||
#print("Total Size: {0}".format(len(md5_per_page)))
|
||||
return md5_per_page
|
||||
|
||||
def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]):
|
||||
@ -278,7 +310,7 @@ class ProbackupTest(object):
|
||||
if self.verbose:
|
||||
print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format(
|
||||
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
|
||||
print(idx_dict)
|
||||
# print(idx_dict)
|
||||
success = False
|
||||
continue
|
||||
if PageNum not in idx_dict['new_pages']:
|
||||
@ -297,7 +329,8 @@ class ProbackupTest(object):
|
||||
if self.verbose:
|
||||
print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format(
|
||||
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
|
||||
print(idx_dict)
|
||||
print("\n Old checksumm: {0}\n New checksumm: {1}".format(idx_dict['old_pages'][PageNum], idx_dict['new_pages'][PageNum]))
|
||||
#print(idx_dict)
|
||||
if PageNum == 0 and idx_dict['type'] == 'spgist':
|
||||
if self.verbose:
|
||||
print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0')
|
||||
@ -309,9 +342,9 @@ class ProbackupTest(object):
|
||||
if self.verbose:
|
||||
print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format(
|
||||
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
|
||||
print(idx_dict)
|
||||
self.assertEqual(success, True, 'Ptrack of index {0} does not correspond to state of its pages.\n Gory Details: \n{1}'.format(
|
||||
idx_dict['type'], idx_dict))
|
||||
#print(idx_dict)
|
||||
#self.assertEqual(success, True, 'Ptrack does not correspond to state of its pages.\n Gory Details: \n{0}'.format(
|
||||
# idx_dict['type'], idx_dict))
|
||||
|
||||
def check_ptrack_recovery(self, idx_dict):
|
||||
size = idx_dict['size']
|
||||
@ -602,7 +635,7 @@ class ProbackupTest(object):
|
||||
|
||||
def switch_wal_segment(self, node):
|
||||
""" Execute pg_switch_wal/xlog() in given node"""
|
||||
if version_to_num(node.safe_psql("postgres", "show server_version")) >= version_to_num('10.0'):
|
||||
if testgres.version_to_num(node.safe_psql("postgres", "show server_version")) >= testgres.version_to_num('10.0'):
|
||||
node.safe_psql("postgres", "select pg_switch_wal()")
|
||||
else:
|
||||
node.safe_psql("postgres", "select pg_switch_xlog()")
|
||||
|
||||
@ -721,11 +721,12 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={'wal_level': 'replica', 'max_wal_senders': '2',
|
||||
'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '1GB',
|
||||
'maintenance_work_mem': '1GB', 'autovacuum': 'off'}
|
||||
'maintenance_work_mem': '1GB', 'autovacuum': 'off', 'full_page_writes': 'off'}
|
||||
)
|
||||
|
||||
self.init_pb(backup_dir)
|
||||
self.add_instance(backup_dir, 'node', node)
|
||||
self.set_archiving(backup_dir, 'node', node)
|
||||
node.start()
|
||||
|
||||
self.create_tblspace_in_node(node, 'somedata')
|
||||
@ -733,17 +734,38 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
|
||||
# CREATE TABLE
|
||||
node.pgbench_init(scale=100, options=['--tablespace=somedata'])
|
||||
# FULL BACKUP
|
||||
self.backup_node(backup_dir, 'node', node, options=["--stream"])
|
||||
#self.backup_node(backup_dir, 'node', node, options=["--stream"])
|
||||
self.backup_node(backup_dir, 'node', node)
|
||||
|
||||
# PTRACK STUFF
|
||||
idx_ptrack = {'type': 'heap'}
|
||||
|
||||
idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts')
|
||||
idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts')
|
||||
idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork(
|
||||
idx_ptrack['path'], idx_ptrack['old_size'])
|
||||
|
||||
pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum'])
|
||||
pgbench.wait()
|
||||
#node.safe_psql("postgres", "update pgbench_accounts set bid = bid +1")
|
||||
node.safe_psql("postgres", "checkpoint")
|
||||
|
||||
idx_ptrack['new_size'] = self.get_fork_size(node, 'pgbench_accounts')
|
||||
idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork(idx_ptrack['path'], idx_ptrack['new_size'])
|
||||
idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
|
||||
node, idx_ptrack['path'], [idx_ptrack['old_size'], idx_ptrack['new_size']])
|
||||
|
||||
self.check_ptrack_sanity(idx_ptrack)
|
||||
## PTRACK STUFF
|
||||
|
||||
# GET LOGICAL CONTENT FROM NODE
|
||||
result = node.safe_psql("postgres", "select * from pgbench_accounts")
|
||||
# FIRTS PTRACK BACKUP
|
||||
self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
|
||||
#self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream"])
|
||||
self.backup_node(backup_dir, 'node', node, backup_type='ptrack')
|
||||
# GET PHYSICAL CONTENT FROM NODE
|
||||
pgdata = self.pgdata_content(node.data_dir)
|
||||
#get_md5_per_page_for_fork
|
||||
|
||||
# RESTORE NODE
|
||||
restored_node = self.make_simple_node(base_dir="{0}/{1}/restored_node".format(module_name, fname))
|
||||
@ -766,10 +788,10 @@ class PtrackBackupTest(ProbackupTest, unittest.TestCase):
|
||||
result_new = restored_node.safe_psql("postgres", "select * from pgbench_accounts")
|
||||
|
||||
# COMPARE RESTORED FILES
|
||||
self.assertEqual(result, result_new)
|
||||
self.assertEqual(result, result_new, 'data is lost')
|
||||
|
||||
if self.paranoia:
|
||||
self.compare_pgdata(pgdata, pgdata_restored)
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
# self.del_test_dir(module_name, fname)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user