db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-07-15 07:04:14 +02:00
Code Issues Releases Activity

tests: table_checksum needs no sorting in fact

Browse Source 
since we are compare table content exactly
This commit is contained in:
Yura Sokolov
2022-12-04 02:26:54 +03:00
parent 8d8a92c1d1
commit 02e3fb0477
6 changed files with 33 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/archive_test.py
View File

@ -2215,7 +2215,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
node.slow_start() node.slow_start()
node.pgbench_init(scale=20) node.pgbench_init(scale=20)
result = node.table_checksum("pgbench_accounts", "aid") result = node.table_checksum("pgbench_accounts")
node.stop() node.stop()
node.cleanup() node.cleanup()
@ -2240,7 +2240,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
node.slow_start() node.slow_start()
result_new = node.table_checksum("pgbench_accounts", "aid") result_new = node.table_checksum("pgbench_accounts")
self.assertEqual(result, result_new) self.assertEqual(result, result_new)

24
tests/cfs_backup_test.py
View File

@ -761,7 +761,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
't_heap', tblspace_name) 't_heap', tblspace_name)
) )
full_result = self.node.table_checksum("t_heap", "id") full_result = self.node.table_checksum("t_heap")
try: try:
backup_id_full = self.backup_node( backup_id_full = self.backup_node(
@ -783,7 +783,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
't_heap') 't_heap')
) )
page_result = self.node.table_checksum("t_heap", "id") page_result = self.node.table_checksum("t_heap")
try: try:
backup_id_page = self.backup_node( backup_id_page = self.backup_node(
@ -824,7 +824,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
self.node.slow_start() self.node.slow_start()
self.assertEqual( self.assertEqual(
full_result, full_result,
self.node.table_checksum("t_heap", "id"), self.node.table_checksum("t_heap"),
'Lost data after restore') 'Lost data after restore')
# CHECK PAGE BACKUP # CHECK PAGE BACKUP
@ -843,7 +843,7 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
self.node.slow_start() self.node.slow_start()
self.assertEqual( self.assertEqual(
page_result, page_result,
self.node.table_checksum("t_heap", "id"), self.node.table_checksum("t_heap"),
'Lost data after restore') 'Lost data after restore')
# @unittest.expectedFailure # @unittest.expectedFailure
@ -877,8 +877,8 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
"FROM generate_series(0,1005000) i".format( "FROM generate_series(0,1005000) i".format(
't_heap_2', tblspace_name_2)) 't_heap_2', tblspace_name_2))
full_result_1 = self.node.table_checksum("t_heap_1", "id") full_result_1 = self.node.table_checksum("t_heap_1")
full_result_2 = self.node.table_checksum("t_heap_2", "id") full_result_2 = self.node.table_checksum("t_heap_2")
try: try:
backup_id_full = self.backup_node( backup_id_full = self.backup_node(
@ -909,8 +909,8 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
't_heap_2') 't_heap_2')
) )
page_result_1 = self.node.table_checksum("t_heap_1", "id") page_result_1 = self.node.table_checksum("t_heap_1")
page_result_2 = self.node.table_checksum("t_heap_2", "id") page_result_2 = self.node.table_checksum("t_heap_2")
try: try:
backup_id_page = self.backup_node( backup_id_page = self.backup_node(
@ -951,11 +951,11 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
self.assertEqual( self.assertEqual(
full_result_1, full_result_1,
self.node.table_checksum("t_heap_1", "id"), self.node.table_checksum("t_heap_1"),
'Lost data after restore') 'Lost data after restore')
self.assertEqual( self.assertEqual(
full_result_2, full_result_2,
self.node.table_checksum("t_heap_2", "id"), self.node.table_checksum("t_heap_2"),
'Lost data after restore') 'Lost data after restore')
# CHECK PAGE BACKUP # CHECK PAGE BACKUP
@ -972,11 +972,11 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
self.assertEqual( self.assertEqual(
page_result_1, page_result_1,
self.node.table_checksum("t_heap_1", "id"), self.node.table_checksum("t_heap_1"),
'Lost data after restore') 'Lost data after restore')
self.assertEqual( self.assertEqual(
page_result_2, page_result_2,
self.node.table_checksum("t_heap_2", "id"), self.node.table_checksum("t_heap_2"),
'Lost data after restore') 'Lost data after restore')
# @unittest.expectedFailure # @unittest.expectedFailure

23
tests/helpers/ptrack_helpers.py
View File

@ -15,7 +15,6 @@ import select
from time import sleep from time import sleep
import re import re
import json import json
from hashlib import md5
import random import random
idx_ptrack = { idx_ptrack = {
@ -202,30 +201,28 @@ class PostgresNodeExtended(testgres.PostgresNode):
os.kill(self.auxiliary_pids[someone][0], sig) os.kill(self.auxiliary_pids[someone][0], sig)
self.is_started = False self.is_started = False
def table_checksum(self, table, sort, dbname="postgres"): def table_checksum(self, table, dbname="postgres"):
curname = "cur_"+str(random.randint(0,2**48))
sum = md5(b"\x01")
con = self.connect(dbname=dbname) con = self.connect(dbname=dbname)
con.execute(f""" curname = "cur_"+str(random.randint(0,2**48))
DECLARE {curname} NO SCROLL CURSOR FOR
SELECT t::text FROM {table} as t ORDER BY {sort};
""")
con.execute("""
DECLARE %s NO SCROLL CURSOR FOR
SELECT t::text FROM %s as t
""" % (curname, table))
sum = hashlib.md5()
while True: while True:
rows = con.execute(f"FETCH FORWARD 5000 FROM {curname}") rows = con.execute("FETCH FORWARD 5000 FROM %s" % curname)
if not rows: if not rows:
break break
for row in rows: for row in rows:
# hash uses SipHash since Python3.4, therefore it is good enough
sum.update(row[0].encode('utf8')) sum.update(row[0].encode('utf8'))
sum.update(b'\x00')
con.execute(f"CLOSE {curname}; ROLLBACK;") con.execute(f"CLOSE {curname}; ROLLBACK;")
con.close() con.close()
sum.update(b'\x02')
return sum.hexdigest() return sum.hexdigest()
class ProbackupTest(object): class ProbackupTest(object):

4
tests/page_test.py
View File

@ -1191,7 +1191,7 @@ class PageTest(ProbackupTest, unittest.TestCase):
pgdata = self.pgdata_content(node.data_dir) pgdata = self.pgdata_content(node.data_dir)
result = node.table_checksum("pgbench_accounts", "aid") result = node.table_checksum("pgbench_accounts")
node_restored = self.make_simple_node( node_restored = self.make_simple_node(
base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) base_dir=os.path.join(self.module_name, self.fname, 'node_restored'))
@ -1203,7 +1203,7 @@ class PageTest(ProbackupTest, unittest.TestCase):
self.set_auto_conf(node_restored, {'port': node_restored.port}) self.set_auto_conf(node_restored, {'port': node_restored.port})
node_restored.slow_start() node_restored.slow_start()
result_new = node_restored.table_checksum("pgbench_accounts", "aid") result_new = node_restored.table_checksum("pgbench_accounts")
self.assertEqual(result, result_new) self.assertEqual(result, result_new)

13
tests/ptrack_test.py
View File

@ -375,7 +375,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
self.switch_wal_segment(node) self.switch_wal_segment(node)
result = node.table_checksum("pgbench_accounts", "aid") result = node.table_checksum("pgbench_accounts")
node_restored.cleanup() node_restored.cleanup()
self.restore_node(backup_dir, 'node', node_restored) self.restore_node(backup_dir, 'node', node_restored)
@ -396,7 +396,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
# Logical comparison # Logical comparison
self.assertEqual( self.assertEqual(
result, result,
node.table_checksum("pgbench_accounts", "aid"), node.table_checksum("pgbench_accounts"),
'Data loss') 'Data loss')
# @unittest.skip("skip") # @unittest.skip("skip")
@ -2037,8 +2037,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
# CREATE TABLE # CREATE TABLE
node.pgbench_init(scale=100, options=['--tablespace=somedata']) node.pgbench_init(scale=100, options=['--tablespace=somedata'])
result = node.table_checksum("pgbench_accounts", "aid", result = node.table_checksum("pgbench_accounts")
dbname="postgres")
# FULL BACKUP # FULL BACKUP
self.backup_node(backup_dir, 'node', node, options=['--stream']) self.backup_node(backup_dir, 'node', node, options=['--stream'])
@ -2075,8 +2074,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
# GET LOGICAL CONTENT FROM NODE # GET LOGICAL CONTENT FROM NODE
# it`s stupid, because hint`s are ignored by ptrack # it`s stupid, because hint`s are ignored by ptrack
result = node.table_checksum("pgbench_accounts", "aid", result = node.table_checksum("pgbench_accounts")
dbname="postgres")
# FIRTS PTRACK BACKUP # FIRTS PTRACK BACKUP
self.backup_node( self.backup_node(
backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) backup_dir, 'node', node, backup_type='ptrack', options=['--stream'])
@ -2109,8 +2107,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase):
restored_node, {'port': restored_node.port}) restored_node, {'port': restored_node.port})
restored_node.slow_start() restored_node.slow_start()
result_new = restored_node.table_checksum("pgbench_accounts", "aid", result_new = restored_node.table_checksum("pgbench_accounts")
dbname="postgres")
# COMPARE RESTORED FILES # COMPARE RESTORED FILES
self.assertEqual(result, result_new, 'data is lost') self.assertEqual(result, result_new, 'data is lost')

4
tests/replica_test.py
View File

@ -326,7 +326,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
self.switch_wal_segment(master) self.switch_wal_segment(master)
before = master.table_checksum("pgbench_accounts", "aid") before = master.table_checksum("pgbench_accounts")
self.validate_pb(backup_dir, 'replica') self.validate_pb(backup_dir, 'replica')
self.assertEqual( self.assertEqual(
@ -342,7 +342,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
node.slow_start() node.slow_start()
# CHECK DATA CORRECTNESS # CHECK DATA CORRECTNESS
after = master.table_checksum("pgbench_accounts", "aid") after = master.table_checksum("pgbench_accounts")
self.assertEqual( self.assertEqual(
before, after, 'Restored data is not equal to original') before, after, 'Restored data is not equal to original')