db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-04-11 11:52:02 +02:00
Code Issues Releases Activity

tests fixes, del_test_dir added

Browse Source
This commit is contained in:
Grigory Smolkin 2017-06-27 08:42:52 +03:00
parent 9c7eae6753
commit 49a8575456
25 changed files with 673 additions and 1060 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tests/__init__.py
View File

@ -6,16 +6,11 @@ from . import init_test, option_test, show_test, \
ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \
ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \
pgpro688, false_positive, replica
false_positive, replica
def load_tests(loader, tests, pattern):
suite = unittest.TestSuite()
suite.addTests(loader.loadTestsFromModule(replica))
# suite.addTests(loader.loadTestsFromModule(pgpro560))
# suite.addTests(loader.loadTestsFromModule(pgpro589))
# suite.addTests(loader.loadTestsFromModule(pgpro688))
# suite.addTests(loader.loadTestsFromModule(false_positive))
suite.addTests(loader.loadTestsFromModule(init_test))
suite.addTests(loader.loadTestsFromModule(option_test))
suite.addTests(loader.loadTestsFromModule(show_test))
@ -33,8 +28,9 @@ def load_tests(loader, tests, pattern):
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate))
suite.addTests(loader.loadTestsFromModule(replica))
suite.addTests(loader.loadTestsFromModule(pgpro560))
suite.addTests(loader.loadTestsFromModule(pgpro589))
suite.addTests(loader.loadTestsFromModule(false_positive))
return suite
# ExpectedFailures are bugs, which should be fixed

92
tests/backup_test.py
View File

@ -1,9 +1,9 @@
import unittest
import os
import six
from time import sleep
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all, clean_all
import shutil
class BackupTest(ProbackupTest, unittest.TestCase):
@ -12,10 +12,6 @@ class BackupTest(ProbackupTest, unittest.TestCase):
super(BackupTest, self).__init__(*args, **kwargs)
self.module_name = 'backup'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
# PGPRO-707
@ -39,8 +35,8 @@ class BackupTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(backup_dir, 'node', node)
show_backup = self.show_pb(backup_dir, 'node')[0]
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("FULL"))
self.assertEqual(show_backup['Status'], "OK")
self.assertEqual(show_backup['Mode'], "FULL")
# postmaster.pid and postmaster.opts shouldn't be copied
excluded = True
@ -56,8 +52,8 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# print self.show_pb(node)
show_backup = self.show_pb(backup_dir, 'node')[1]
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("PAGE"))
self.assertEqual(show_backup['Status'], "OK")
self.assertEqual(show_backup['Mode'], "PAGE")
# Check parent backup
self.assertEqual(
@ -68,15 +64,16 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
show_backup = self.show_pb(backup_dir, 'node')[2]
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("PTRACK"))
self.assertEqual(show_backup['Status'], "OK")
self.assertEqual(show_backup['Mode'], "PTRACK")
# Check parent backup
self.assertEqual(
page_backup_id,
self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"])
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_smooth_checkpoint(self):
@ -93,9 +90,12 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node.start()
self.backup_node(backup_dir, 'node' ,node, options=["-C"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
#@unittest.skip("skip")
def test_incremental_backup_without_full(self):
"""page-level backup without validated full backup"""
@ -115,7 +115,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -127,16 +127,17 @@ class BackupTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR"))
node.stop()
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR")
@unittest.expectedFailure
# Need to forcibly validate parent
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.expectedFailure
def test_incremental_backup_corrupt_full(self):
"""page-level backup with corrupted full backup"""
fname = self.id().split('.')[3]
@ -151,29 +152,37 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node.start()
backup_id = self.backup_node(backup_dir, 'node', node)
file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf")
file = os.path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf")
os.remove(file)
try:
self.validate_pb(backup_dir, 'node')
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of validation of corrupted backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue("INFO: Validate backups of the instance 'node'\n" in e.message
and 'WARNING: Backup file "{0}" is not found\n'.format(file) in e.message
and "WARNING: Backup {0} is corrupted\n".format(backup_id) in e.message
and "INFO: Some backups are not valid\n" in e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
try:
self.backup_node(backup_dir, 'node', node, backup_type="page")
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
"ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n",
"\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
sleep(1)
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# sleep(1)
self.assertEqual(self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT")
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR")
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR"))
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_ptrack_threads(self):
@ -190,12 +199,13 @@ class BackupTest(ProbackupTest, unittest.TestCase):
node.start()
self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_ptrack_threads_stream(self):
@ -213,7 +223,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"])
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK"))
node.stop()
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK")
# Clean after yourself
self.del_test_dir(self.module_name, fname)

35
tests/delete_test.py
View File

@ -1,9 +1,9 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all, clean_all
import subprocess
import shutil
class DeleteTest(ProbackupTest, unittest.TestCase):
@ -12,10 +12,6 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
super(DeleteTest, self).__init__(*args, **kwargs)
self.module_name = 'delete'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_delete_full_backups(self):
@ -55,7 +51,8 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
self.assertEqual(show_backups[0]['ID'], id_1)
self.assertEqual(show_backups[1]['ID'], id_3)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_delete_increment_page(self):
"""delete increment and all after him"""
@ -88,12 +85,13 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 2)
self.assertEqual(show_backups[0]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[0]['Status'], six.b("OK"))
self.assertEqual(show_backups[1]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[1]['Status'], six.b("OK"))
self.assertEqual(show_backups[0]['Mode'], "FULL")
self.assertEqual(show_backups[0]['Status'], "OK")
self.assertEqual(show_backups[1]['Mode'], "FULL")
self.assertEqual(show_backups[1]['Status'], "OK")
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_delete_increment_ptrack(self):
"""delete increment and all after him"""
@ -126,9 +124,10 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 2)
self.assertEqual(show_backups[0]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[0]['Status'], six.b("OK"))
self.assertEqual(show_backups[1]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[1]['Status'], six.b("OK"))
self.assertEqual(show_backups[0]['Mode'], "FULL")
self.assertEqual(show_backups[0]['Status'], "OK")
self.assertEqual(show_backups[1]['Mode'], "FULL")
self.assertEqual(show_backups[1]['Status'], "OK")
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

181
tests/false_positive.py
View File

@ -1,11 +1,10 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from datetime import datetime, timedelta
from testgres import stop_all
from testgres import stop_all, clean_all
import subprocess
from sys import exit
import shutil
class FalsePositive(ProbackupTest, unittest.TestCase):
@ -14,10 +13,6 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
super(FalsePositive, self).__init__(*args, **kwargs)
self.module_name = 'false_positive'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_pgpro561(self):
@ -26,62 +21,64 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
check that archiving is not successful on node1
"""
fname = self.id().split('.')[3]
master = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/master".format(fname),
set_archiving=True,
node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
master.start()
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node1', node1)
self.set_archiving(backup_dir, 'node1', node1)
node1.start()
self.assertEqual(self.init_pb(master), six.b(""))
id = self.backup_pb(master, backup_type='full', options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node1', node1, options=["--stream"])
node1 = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/node1".format(fname))
node1.cleanup()
node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname))
node2.cleanup()
master.psql(
node1.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
self.backup_pb(master, backup_type='page', options=["--stream"])
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir)
node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port))
node1.start({"-t": "600"})
self.backup_node(backup_dir, 'node1', node1, backup_type='page', options=["--stream"])
self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir)
node2.append_conf('postgresql.auto.conf', 'port = {0}'.format(node2.port))
node2.start({"-t": "600"})
timeline_master = master.get_control_data()["Latest checkpoint's TimeLineID"]
timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"]
self.assertEqual(timeline_master, timeline_node1, "Timelines on Master and Node1 should be equal. This is unexpected")
timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"]
self.assertEqual(timeline_node1, timeline_node2, "Timelines on Master and Node1 should be equal. This is unexpected")
archive_command_master = master.safe_psql("postgres", "show archive_command")
archive_command_node1 = node1.safe_psql("postgres", "show archive_command")
self.assertEqual(archive_command_master, archive_command_node1, "Archive command on Master and Node should be equal. This is unexpected")
archive_command_node2 = node2.safe_psql("postgres", "show archive_command")
self.assertEqual(archive_command_node1, archive_command_node2, "Archive command on Master and Node should be equal. This is unexpected")
res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL")
result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL")
# self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip()))
if res == six.b(""):
if result == "":
self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command')
master.stop()
node1.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def pgpro688(self):
"""
make node with archiving, make backup,
get Recovery Time, validate to Recovery Time
Waiting PGPRO-688
"""
"""make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
id = self.backup_pb(node, backup_type='full')
recovery_time = self.show_pb(node, id=id)['recovery-time']
backup_id = self.backup_node(backup_dir, 'node', node)
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
# Uncommenting this section will make this test True Positive
#node.psql("postgres", "select pg_create_restore_point('123')")
@ -89,46 +86,58 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
#node.psql("postgres", "select pg_switch_xlog()")
####
try:
self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])
self.assertEqual(1, 0, 'Error is expected because We should not be able safely validate "Recovery Time" without wal record with timestamp')
except ProbackupException, e:
self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message)
#try:
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(recovery_time)])
# we should die here because exception is what we expect to happen
# self.assertEqual(1, 0, "Expecting Error because it should not be possible safely validate 'Recovery Time' without wal record with timestamp.\n Output: {0} \n CMD: {1}".format(
# repr(self.output), self.cmd))
# except ProbackupException as e:
# self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message,
# '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def pgpro702_688(self):
"""
make node without archiving, make stream backup,
get Recovery Time, validate to Recovery Time
"""
"""make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
id = self.backup_pb(node, backup_type='full', options=["--stream"])
recovery_time = self.show_pb(node, id=id)['recovery-time']
backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"])
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, options=["--time='{0}'".format(recovery_time)]))
self.validate_pb(backup_dir, 'node', node, options=["--time='{0}'".format(recovery_time)]))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
@unittest.expectedFailure
def test_validate_wal_lost_segment(self):
"""Loose segment located between backups. ExpectedFailure. This is BUG """
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, backup_type='full')
self.backup_node(backup_dir, 'node', node)
# make some wals
node.pgbench_init(scale=2)
@ -141,15 +150,59 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
# delete last wal segment
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, "wal", 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals = map(int, wals)
os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals))))
os.remove(os.path.join(wals_dir, '0000000' + str(max(wals))))
##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it
##### We need archive-push ASAP
self.backup_pb(node, backup_type='full')
self.assertTrue('validation completed successfully' in self.validate_pb(node))
self.backup_node(backup_dir, 'node', node)
self.assertFalse('validation completed successfully' in self.validate_pb(backup_dir, 'node'))
########
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
@unittest.expectedFailure
# Need to force validation of ancestor-chain
def test_incremental_backup_corrupt_full_1(self):
"""page-level backup with corrupted full backup"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
backup_id = self.backup_node(backup_dir, 'node', node)
file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf")
os.remove(file)
try:
self.backup_node(backup_dir, 'node', node, backup_type="page")
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
sleep(1)
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR")
# Clean after yourself
self.del_test_dir(self.module_name, fname)

153
tests/helpers/ptrack_helpers.py
View File

@ -1,10 +1,10 @@
# you need os for unittest to work
import os
from sys import exit
from sys import exit, argv, version_info
import subprocess
import shutil
import six
from testgres import get_new_node
from testgres import get_new_node, clean_all
import hashlib
import re
import pwd
@ -73,40 +73,6 @@ def dir_files(base_dir):
out_list.sort()
return out_list
class ShowBackup(object):
def __init__(self, line):
self.counter = 0
print split_line
self.id = self.get_inc(split_line)
# TODO: parse to datetime
if len(split_line) == 12:
self.recovery_time = "%s %s" % (self.get_inc(split_line), self.get_inc(split_line))
# if recovery time is '----'
else:
self.recovery_time = self.get_inc(split_line)
self.mode = self.get_inc(split_line)
# print self.mode
self.wal = self.get_inc(split_line)
self.cur_tli = self.get_inc(split_line)
# slash
self.counter += 1
self.parent_tli = self.get_inc(split_line)
# TODO: parse to interval
self.time = self.get_inc(split_line)
# TODO: maybe rename to size?
self.data = self.get_inc(split_line)
self.start_lsn = self.get_inc(split_line)
self.stop_lsn = self.get_inc(split_line)
self.status = self.get_inc(split_line)
def get_inc(self, split_line):
# self.counter += 1
# return split_line[self.counter - 1]
return split_line
class ProbackupTest(object):
def __init__(self, *args, **kwargs):
super(ProbackupTest, self).__init__(*args, **kwargs)
@ -134,7 +100,6 @@ class ProbackupTest(object):
self.test_env["LC_MESSAGES"] = "C"
self.test_env["LC_TIME"] = "C"
self.helpers_path = os.path.dirname(os.path.realpath(__file__))
self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir))
self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs'))
@ -145,6 +110,10 @@ class ProbackupTest(object):
self.probackup_path = os.path.abspath(os.path.join(
self.dir_path, "../pg_probackup"))
self.user = self.get_username()
if '-v' in argv or '--verbose' in argv:
self.verbose = True
else:
self.verbose = False
def arcwal_dir(self, node):
return "%s/backup/wal" % node.base_dir
@ -228,10 +197,12 @@ class ProbackupTest(object):
byte_size_minus_header = byte_size - header_size
file = os.open(file + '_ptrack', os.O_RDONLY)
os.lseek(file, header_size, 0)
lot_of_bytes = os.read(file, byte_size_minus_header)
for byte in lot_of_bytes:
lots_of_bytes = os.read(file, byte_size_minus_header)
byte_list = [lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes))]
for byte in byte_list:
#byte_inverted = bin(int(byte, base=16))[2:][::-1]
#bits = (byte >> x) & 1 for x in range(7, -1, -1)
byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1]
# byte_to_bits = (byte >> x) & 1 for x in range(7, -1, -1)
for bit in byte_inverted:
if len(ptrack_bits_for_fork) < size:
ptrack_bits_for_fork.append(int(bit))
@ -249,9 +220,10 @@ class ProbackupTest(object):
# Page was not present before, meaning that relation got bigger
# Ptrack should be equal to 1
if idx_dict['ptrack'][PageNum] != 1:
print 'Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
print idx_dict
if self.verbose:
print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict)
success = False
continue
if PageNum not in idx_dict['new_pages']:
@ -266,29 +238,34 @@ class ProbackupTest(object):
if idx_dict['new_pages'][PageNum] != idx_dict['old_pages'][PageNum]:
# Page has been changed, meaning that ptrack should be equal to 1
if idx_dict['ptrack'][PageNum] != 1:
print 'Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
print idx_dict
if self.verbose:
print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict)
if PageNum == 0 and idx_dict['type'] == 'spgist':
print 'SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0'
if self.verbose:
print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0')
continue
success = False
else:
# Page has not been changed, meaning that ptrack should be equal to 0
if idx_dict['ptrack'][PageNum] != 0:
print 'Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
print idx_dict
self.assertEqual(success, True)
if self.verbose:
print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict)
self.assertEqual(success, True, 'Ptrack of index {0} does not correspond to state of its pages.\n Gory Details: \n{1}'.format(
idx_dict['type'], idx_dict))
def check_ptrack_recovery(self, idx_dict):
success = True
size = idx_dict['size']
for PageNum in range(size):
if idx_dict['ptrack'][PageNum] != 1:
print 'Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
print idx_dict
if self.verbose:
print('Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict)
success = False
self.assertEqual(success, True)
@ -296,29 +273,23 @@ class ProbackupTest(object):
success = True
for PageNum in range(size):
if idx_dict['ptrack'][PageNum] != 0:
print 'Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
print idx_dict
if self.verbose:
print('Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format(
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]))
print(idx_dict)
success = False
self.assertEqual(success, True)
self.assertEqual(success, True, '')
def run_pb(self, command, async=False):
def run_pb(self, command):
try:
self.cmd = [' '.join(map(str,[self.probackup_path] + command))]
print self.cmd
if async is True:
return subprocess.Popen(
[self.probackup_path] + command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.test_env
)
else:
self.output = subprocess.check_output(
[self.probackup_path] + command,
stderr=subprocess.STDOUT,
env=self.test_env
)
if self.verbose:
print(self.cmd)
self.output = subprocess.check_output(
[self.probackup_path] + command,
stderr=subprocess.STDOUT,
env=self.test_env
).decode("utf-8")
if command[0] == 'backup':
# return backup ID
for line in self.output.splitlines():
@ -327,7 +298,7 @@ class ProbackupTest(object):
else:
return self.output
except subprocess.CalledProcessError as e:
raise ProbackupException(e.output, self.cmd)
raise ProbackupException(e.output.decode("utf-8"), self.cmd)
def init_pb(self, backup_dir):
@ -358,12 +329,21 @@ class ProbackupTest(object):
def clean_pb(self, backup_dir):
shutil.rmtree(backup_dir, ignore_errors=True)
def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False):
def backup_node(self, backup_dir, instance, node=False, data_dir=False, backup_type="full", options=[]):
if not node and not data_dir:
print('You must provide ether node or data_dir for backup')
exit(1)
if node:
pgdata = node.data_dir
if data_dir:
pgdata = data_dir
cmd_list = [
"backup",
"-B", backup_dir,
"-D", node.data_dir,
"-D", pgdata,
"-p", "%i" % node.port,
"-d", "postgres",
"--instance={0}".format(instance)
@ -371,7 +351,7 @@ class ProbackupTest(object):
if backup_type:
cmd_list += ["-b", backup_type]
return self.run_pb(cmd_list + options, async)
return self.run_pb(cmd_list + options)
def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]):
if data_dir is None:
@ -429,9 +409,9 @@ class ProbackupTest(object):
if i == '':
backup_record_split.remove(i)
if len(header_split) != len(backup_record_split):
print warning.format(
print(warning.format(
header=header, body=body,
header_split=header_split, body_split=backup_record_split)
header_split=header_split, body_split=backup_record_split))
exit(1)
new_dict = dict(zip(header_split, backup_record_split))
backup_list.append(new_dict)
@ -473,7 +453,6 @@ class ProbackupTest(object):
if backup_id:
cmd_list += ["-i", backup_id]
# print(cmd_list)
return self.run_pb(cmd_list + options)
def delete_expired(self, backup_dir, instance, options=[]):
@ -563,3 +542,17 @@ class ProbackupTest(object):
def get_username(self):
""" Returns current user name """
return pwd.getpwuid(os.getuid())[0]
def del_test_dir(self, module_name, fname):
""" Returns current user name """
try:
clean_all()
except:
pass
shutil.rmtree(os.path.join(self.tmp_path, self.module_name, fname),
ignore_errors=True)
try:
os.rmdir(os.path.join(self.tmp_path, self.module_name))
except:
pass

28
tests/init_test.py
View File

@ -2,8 +2,9 @@ import unittest
from sys import exit
import os
from os import path
import six
from helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException
from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException
from testgres import stop_all, clean_all
import shutil
class InitTest(ProbackupTest, unittest.TestCase):
@ -25,20 +26,22 @@ class InitTest(ProbackupTest, unittest.TestCase):
['backups', 'wal']
)
self.add_instance(backup_dir, 'node', node)
self.assertEqual("INFO: Instance 'node' successfully deleted\n",
self.del_instance(backup_dir, 'node', node),
self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node', node),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
try:
self.show_pb(backup_dir, 'node')
self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
"ERROR: Instance 'node' does not exist in this backup catalog\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_already_exist(self):
"""Failure with backup catalog already existed"""
fname = self.id().split(".")[3]
@ -49,11 +52,15 @@ class InitTest(ProbackupTest, unittest.TestCase):
self.show_pb(backup_dir, 'node')
self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
"ERROR: Instance 'node' does not exist in this backup catalog\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_abs_path(self):
"""failure with backup catalog should be given as absolute path"""
fname = self.id().split(".")[3]
@ -63,11 +70,10 @@ class InitTest(ProbackupTest, unittest.TestCase):
self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)])
self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
"ERROR: -B, --backup-path must be an absolute path\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

51
tests/option_test.py
View File

@ -1,8 +1,8 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all, clean_all
import shutil
class OptionTest(ProbackupTest, unittest.TestCase):
@ -11,10 +11,6 @@ class OptionTest(ProbackupTest, unittest.TestCase):
super(OptionTest, self).__init__(*args, **kwargs)
self.module_name = 'option'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_help_1(self):
@ -24,7 +20,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out:
self.assertEqual(
self.run_pb(["--help"]),
help_out.read()
help_out.read().decode("utf-8")
)
# @unittest.skip("skip")
@ -35,7 +31,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out:
self.assertEqual(
self.run_pb(["--version"]),
version_out.read()
version_out.read().decode("utf-8")
)
# @unittest.skip("skip")
@ -47,7 +43,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
self.run_pb(["backup", "-b", "full"])
self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -59,7 +55,6 @@ class OptionTest(ProbackupTest, unittest.TestCase):
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'})
try:
node.stop()
except:
@ -73,7 +68,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"])
self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: required parameter not specified: --instance\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -83,7 +78,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir])
self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -93,23 +88,25 @@ class OptionTest(ProbackupTest, unittest.TestCase):
self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"])
self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: invalid backup-mode "bad"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# delete failure without ID
try:
self.run_pb(["delete", "-B", backup_dir, "--instance=node"])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: required backup ID not specified\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
#@unittest.skip("skip")
def test_options_5(self):
"""check options test"""
@ -118,7 +115,8 @@ class OptionTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'})
self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir)))
self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir),
self.init_pb(backup_dir))
self.add_instance(backup_dir, 'node', node)
node.start()
@ -131,7 +129,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: syntax error in " = INFINITE"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -149,7 +147,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: invalid backup-mode ""\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -162,10 +160,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write("retention-redundancy=1\n")
self.assertEqual(
self.show_config(backup_dir, 'node')['retention-redundancy'],
six.b('1')
)
self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1')
# User cannot send --system-identifier parameter via command line
try:
@ -173,7 +168,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: option system-identifier cannot be specified in command line\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -187,7 +182,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
"ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -205,10 +200,10 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: invalid option "TIMELINEID"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# self.clean_pb(backup_dir)
# node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

304
tests/pb_lib.py
View File

@ -1,304 +0,0 @@
import os
from os import path, listdir
import subprocess
import shutil
import six
from testgres import get_new_node
def dir_files(base_dir):
out_list = []
for dir_name, subdir_list, file_list in os.walk(base_dir):
if dir_name != base_dir:
out_list.append(path.relpath(dir_name, base_dir))
for fname in file_list:
out_list.append(path.relpath(path.join(dir_name, fname), base_dir))
out_list.sort()
return out_list
class ShowBackup(object):
def __init__(self, split_line):
self.counter = 0
self.id = self.get_inc(split_line)
# TODO: parse to datetime
if len(split_line) == 12:
self.recovery_time = "%s %s" % (self.get_inc(split_line),
self.get_inc(split_line))
# if recovery time is '----'
else:
self.recovery_time = self.get_inc(split_line)
self.mode = self.get_inc(split_line)
self.cur_tli = self.get_inc(split_line)
# slash
self.counter += 1
self.parent_tli = self.get_inc(split_line)
# TODO: parse to interval
self.time = self.get_inc(split_line)
# TODO: maybe rename to size?
self.data = self.get_inc(split_line)
self.start_lsn = self.get_inc(split_line)
self.stop_lsn = self.get_inc(split_line)
self.status = self.get_inc(split_line)
def get_inc(self, split_line):
self.counter += 1
return split_line[self.counter - 1]
class ProbackupTest(object):
def __init__(self, *args, **kwargs):
super(ProbackupTest, self).__init__(*args, **kwargs)
self.test_env = os.environ.copy()
envs_list = [
"LANGUAGE",
"LC_ALL",
"PGCONNECT_TIMEOUT",
"PGDATA",
"PGDATABASE",
"PGHOSTADDR",
"PGREQUIRESSL",
"PGSERVICE",
"PGSSLMODE",
"PGUSER",
"PGPORT",
"PGHOST"
]
for e in envs_list:
try:
del self.test_env[e]
except:
pass
self.test_env["LC_MESSAGES"] = "C"
self.test_env["LC_TIME"] = "C"
self.dir_path = path.dirname(os.path.realpath(__file__))
try:
os.makedirs(path.join(self.dir_path, "tmp_dirs"))
except:
pass
self.probackup_path = os.path.abspath(path.join(
self.dir_path,
"../pg_probackup"
))
def arcwal_dir(self, node):
return "%s/backup/wal" % node.base_dir
def backup_dir(self, node):
return os.path.abspath("%s/backup" % node.base_dir)
def make_bnode(self, base_dir=None, allows_streaming=False, options={}):
real_base_dir = path.join(self.dir_path, base_dir)
shutil.rmtree(real_base_dir, ignore_errors=True)
node = get_new_node('test', base_dir=real_base_dir)
node.init(allows_streaming=allows_streaming)
if not allows_streaming:
node.append_conf("postgresql.conf", "wal_level = hot_standby")
node.append_conf("postgresql.conf", "archive_mode = on")
node.append_conf(
"postgresql.conf",
"""archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node))
)
for key, value in six.iteritems(options):
node.append_conf("postgresql.conf", "%s = %s" % (key, value))
return node
def make_bnode_replica(self, root_node, base_dir=None, options={}):
real_base_dir = path.join(self.dir_path, base_dir)
shutil.rmtree(real_base_dir, ignore_errors=True)
root_node.backup("basebackup")
replica = get_new_node("replica", base_dir=real_base_dir)
# replica.init_from_backup(root_node, "data_replica", has_streaming=True)
# Move data from backup
backup_path = os.path.join(root_node.base_dir, "basebackup")
shutil.move(backup_path, replica.data_dir)
os.chmod(replica.data_dir, 0o0700)
# Change port in config file
replica.append_conf(
"postgresql.conf",
"port = {}".format(replica.port)
)
# Enable streaming
replica.enable_streaming(root_node)
for key, value in six.iteritems(options):
replica.append_conf("postgresql.conf", "%s = %s" % (key, value))
return replica
def run_pb(self, command):
try:
return subprocess.check_output(
[self.probackup_path] + command,
stderr=subprocess.STDOUT,
env=self.test_env
)
except subprocess.CalledProcessError as err:
return err.output
def init_pb(self, node):
return self.run_pb([
"init",
"-B", self.backup_dir(node),
"-D", node.data_dir
])
def clean_pb(self, node):
shutil.rmtree(self.backup_dir(node), ignore_errors=True)
def backup_pb(self, node, backup_type="full", options=[]):
cmd_list = [
"backup",
"-D", node.data_dir,
"-B", self.backup_dir(node),
"-p", "%i" % node.port,
"-d", "postgres"
]
if backup_type:
cmd_list += ["-b", backup_type]
return self.run_pb(cmd_list + options)
def backup_pb_proc(self, node, backup_dir, backup_type="full",
stdout=None, stderr=None, options=[]):
cmd_list = [
self.probackup_path,
"backup",
"-D", node.data_dir,
"-B", backup_dir,
"-p", "%i" % (node.port),
"-d", "postgres"
]
if backup_type:
cmd_list += ["-b", backup_type]
proc = subprocess.Popen(
cmd_list + options,
stdout=stdout,
stderr=stderr
)
return proc
def restore_pb(self, node, id=None, options=[]):
cmd_list = [
"-D", node.data_dir,
"-B", self.backup_dir(node),
"restore"
]
if id:
cmd_list.append(id)
# print(cmd_list)
return self.run_pb(cmd_list + options)
def show_pb(self, node, id=None, options=[], as_text=False):
cmd_list = [
"show",
"-B", self.backup_dir(node),
]
if id:
cmd_list += [id]
# print(cmd_list)
if as_text:
return self.run_pb(options + cmd_list)
elif id is None:
return [ShowBackup(line.split()) for line in self.run_pb(options + cmd_list).splitlines()[3:]]
else:
return dict([
line.split(six.b("="))
for line in self.run_pb(options + cmd_list).splitlines()
if line[0] != six.b("#")[0]
])
def validate_pb(self, node, id=None, options=[]):
cmd_list = [
"-B", self.backup_dir(node),
"validate",
]
if id:
cmd_list += [id]
# print(cmd_list)
return self.run_pb(options + cmd_list)
def delete_pb(self, node, id=None, options=[]):
cmd_list = [
"-B", self.backup_dir(node),
"delete",
]
if id:
cmd_list += [id]
# print(cmd_list)
return self.run_pb(options + cmd_list)
def retention_purge_pb(self, node, options=[]):
cmd_list = [
"-B", self.backup_dir(node),
"retention", "purge",
]
return self.run_pb(options + cmd_list)
def retention_show(self, node, options=[]):
cmd_list = [
"-B", self.backup_dir(node),
"retention", "show",
]
return self.run_pb(options + cmd_list)
def get_control_data(self, node):
pg_controldata = node.get_bin_path("pg_controldata")
out_data = {}
lines = subprocess.check_output(
[pg_controldata] + ["-D", node.data_dir],
stderr=subprocess.STDOUT,
env=self.test_env
).splitlines()
for l in lines:
key, value = l.split(b":", maxsplit=1)
out_data[key.strip()] = value.strip()
return out_data
def get_recovery_conf(self, node):
out_dict = {}
with open(path.join(node.data_dir, "recovery.conf"), "r") as recovery_conf:
for line in recovery_conf:
try:
key, value = line.split("=")
except:
continue
out_dict[key.strip()] = value.strip(" '").replace("'\n", "")
return out_dict
def wrong_wal_clean(self, node, wal_size):
wals_dir = path.join(self.backup_dir(node), "wal")
wals = [f for f in listdir(wals_dir) if path.isfile(path.join(wals_dir, f))]
wals.sort()
file_path = path.join(wals_dir, wals[-1])
if path.getsize(file_path) != wal_size:
os.remove(file_path)
def guc_wal_segment_size(self, node):
var = node.execute("postgres", "select setting from pg_settings where name = 'wal_segment_size'")
return int(var[0][0]) * self.guc_wal_block_size(node)
def guc_wal_block_size(self, node):
var = node.execute("postgres", "select setting from pg_settings where name = 'wal_block_size'")
return int(var[0][0])

62
tests/pgpro560.py
View File

@ -1,21 +1,17 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
import unittest
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
from datetime import datetime, timedelta
from testgres import stop_all
from testgres import stop_all, clean_all
import subprocess
from sys import exit
import shutil
class CheckSystemID(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(CheckSystemID, self).__init__(*args, **kwargs)
@classmethod
def tearDownClass(cls):
stop_all()
self.module_name = 'pgpro560'
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -27,24 +23,32 @@ class CheckSystemID(ProbackupTest, unittest.TestCase):
check that backup failed
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
file = os.path.join(node.base_dir,'data', 'global', 'pg_control')
os.remove(file)
try:
self.backup_pb(node, backup_type='full', options=['--stream'])
assertEqual(1, 0, 'Error is expected because of control file loss')
except ProbackupException, e:
self.backup_node(backup_dir, 'node', node, options=['--stream'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: could not open file' and 'pg_control' in e.message,
'Expected error is about control file loss')
'ERROR: could not open file' in e.message
and 'pg_control' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_pgpro560_systemid_mismatch(self):
"""
@ -54,25 +58,33 @@ class CheckSystemID(ProbackupTest, unittest.TestCase):
check that backup failed
"""
fname = self.id().split('.')[3]
node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node1".format(fname),
node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
node1.start()
node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node2".format(fname),
node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
node2.start()
self.assertEqual(self.init_pb(node1), six.b(""))
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node1', node1)
try:
self.backup_pb(node1, data_dir=node2.data_dir, backup_type='full', options=['--stream'])
assertEqual(1, 0, 'Error is expected because of SYSTEM ID mismatch')
except ProbackupException, e:
self.backup_node(backup_dir, 'node1', node1, data_dir=node2.data_dir, options=['--stream'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: Backup data directory was initialized for system id' and
'but target backup directory system id is' in e.message,
'Expected error is about SYSTEM ID mismatch')
'ERROR: Backup data directory was initialized for system id' in e.message
and 'but target backup directory system id is' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)

78
tests/pgpro589.py
View File

@ -1,21 +1,17 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
import unittest
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
from datetime import datetime, timedelta
from testgres import stop_all
from testgres import stop_all, clean_all
import subprocess
from sys import exit
import shutil
class ArchiveCheck(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ArchiveCheck, self).__init__(*args, **kwargs)
@classmethod
def tearDownClass(cls):
stop_all()
self.module_name = 'pgpro589'
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -26,10 +22,13 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
check ERROR text
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
node.pgbench_init(scale=5)
@ -41,15 +40,17 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
pgbench.wait()
pgbench.stdout.close()
path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip()
self.assertEqual(self.init_pb(node), six.b(""))
try:
self.backup_pb(node, backup_type='full', options=['--archive-timeout=10'])
assertEqual(1, 0, 'Error is expected because of disabled archive_mode')
except ProbackupException, e:
self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n')
self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of disabled archive_mode.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_pgpro589(self):
"""
@ -59,12 +60,16 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
check that no files where copied to backup catalogue
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
node.append_conf("postgresql.auto.conf", "archive_mode = on")
node.append_conf("postgresql.auto.conf", "wal_level = archive")
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
# make erroneus archive_command
node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'")
node.start()
@ -76,22 +81,25 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase):
)
pgbench.wait()
pgbench.stdout.close()
path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip()
self.assertEqual(self.init_pb(node), six.b(""))
path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip().decode("utf-8")
try:
self.backup_pb(
node, backup_type='full', options=['--archive-timeout=10'])
assertEqual(1, 0, 'Error is expected because of missing archive wal segment with start_backup() LSN')
except ProbackupException, e:
self.assertTrue('INFO: wait for LSN' in e.message, "Expecting 'INFO: wait for LSN'")
self.assertTrue('ERROR: switched WAL segment' and 'could not be archived' in e.message,
"Expecting 'ERROR: switched WAL segment could not be archived'")
self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10'])
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because of missing archive wal segment with start_lsn.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'INFO: wait for LSN' in e.message
and 'ERROR: switched WAL segment' in e.message
and 'could not be archived' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
id = self.show_pb(node)[0]['ID']
self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status')
#print self.backup_dir(node)
file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path)
backup_id = self.show_pb(backup_dir, 'node')[0]['ID']
self.assertEqual('ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status')
file = os.path.join(backup_dir, 'backups', 'node', backup_id, 'database', path)
self.assertFalse(os.path.isfile(file),
'\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file))
# Clean after yourself
self.del_test_dir(self.module_name, fname)

201
tests/pgpro688.py
View File

@ -1,201 +0,0 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from datetime import datetime, timedelta
from testgres import stop_all, get_username
import subprocess
from sys import exit, _getframe
import shutil
import time
class ReplicaTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ReplicaTest, self).__init__(*args, **kwargs)
self.module_name = 'replica'
self.instance_master = 'master'
self.instance_replica = 'replica'
# @classmethod
# def tearDownClass(cls):
# stop_all()
@unittest.skip("skip")
# @unittest.expectedFailure
def test_replica_stream_full_backup(self):
"""make full stream backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, self.instance_master, master)
master.start()
# Make empty Object 'replica' from new node
replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname))
replica_port = replica.port
replica.cleanup()
# FULL STREAM backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream'])
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
# FULL STREAM backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream'])
# Restore last backup from master to Replica directory
self.restore_node(backup_dir, self.instance_master, replica.data_dir)
# Set Replica
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port))
replica.start({"-t": "600"})
# Check replica
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Add instance replica
self.add_instance(backup_dir, self.instance_replica, replica)
# FULL STREAM backup of replica
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[
'--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
# Validate instance replica
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
def test_replica_archive_full_backup(self):
"""make page archive backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'}
)
self.set_archiving(backup_dir, self.instance_master, master)
self.init_pb(backup_dir)
self.add_instance(backup_dir, self.instance_master, master)
master.start()
# Make empty Object 'replica' from new node
replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname))
replica_port = replica.port
replica.cleanup()
# FULL ARCHIVE backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full')
# Create table t_heap
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
# PAGE ARCHIVE backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='page')
# Restore last backup from master to Replica directory
self.restore_node(backup_dir, self.instance_master, replica.data_dir)
# Set Replica
self.set_archiving(backup_dir, self.instance_replica, replica, replica=True)
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port))
replica.start({"-t": "600"})
# Check replica
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make FULL ARCHIVE backup from replica
self.add_instance(backup_dir, self.instance_replica, replica)
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[
'--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
# Drop Table t_heap
after = master.execute("postgres", "drop table t_heap")
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
# Make page backup from replica
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[
'--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
@unittest.skip("skip")
def test_replica_archive_full_backup_123(self):
"""
make full archive backup from replica
"""
fname = self.id().split('.')[3]
master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
master.append_conf('postgresql.auto.conf', 'archive_timeout = 10')
master.start()
replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname))
replica_port = replica.port
replica.cleanup()
self.assertEqual(self.init_pb(master), six.b(""))
self.backup_pb(node=master, backup_type='full', options=['--stream'])
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
id = self.backup_pb(master, backup_type='page', options=['--stream'])
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir)
# Settings for Replica
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
# Set Archiving for replica
self.set_archiving_conf(replica, replica=True)
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port))
replica.start({"-t": "600"})
# Replica Started
# master.execute("postgres", "checkpoint")
# Check replica
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make backup from replica
self.assertEqual(self.init_pb(replica), six.b(""))
self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30'])
self.validate_pb(replica)

18
tests/ptrack_clean.py
View File

@ -1,8 +1,8 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import unittest
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,9 +10,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_clean'
def teardown(self):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_ptrack_clean(self):
@ -89,8 +86,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack bits are cleaned
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size'])
print self.show_pb(backup_dir, 'node', as_text=True)
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

30
tests/ptrack_cluster.py
View File

@ -1,8 +1,8 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import unittest
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,10 +10,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_cluster'
def teardown(self):
# clean_all()
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_ptrack_cluster_btree(self):
@ -72,7 +68,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_ptrack_cluster_spgist(self):
fname = self.id().split('.')[3]
@ -130,7 +127,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_ptrack_cluster_brin(self):
fname = self.id().split('.')[3]
@ -188,7 +186,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_ptrack_cluster_gist(self):
fname = self.id().split('.')[3]
@ -246,7 +245,8 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
def test_ptrack_cluster_gin(self):
fname = self.id().split('.')[3]
@ -304,7 +304,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

23
tests/ptrack_move_to_tablespace.py
View File

@ -1,10 +1,8 @@
import unittest
from sys import exit
from testgres import get_new_node, stop_all
import os
from signal import SIGTERM
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from time import sleep
import unittest
from testgres import clean_all, stop_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -12,9 +10,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_move_to_tablespace'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -58,7 +57,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack has correct bits after recovery
self.check_ptrack_recovery(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

29
tests/ptrack_recovery.py
View File

@ -1,20 +1,20 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
import os
from signal import SIGTERM
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from time import sleep
from testgres import clean_all, stop_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_move_to_tablespace'
self.module_name = 'ptrack_recovery'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -45,12 +45,13 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# get path to heap and index files
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
print 'Killing postmaster. Losing Ptrack changes'
if self.verbose:
print('Killing postmaster. Losing Ptrack changes')
node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)})
if not node.status():
node.start()
else:
print "Die! Die! Why won't you die?... Why won't you die?"
print("Die! Die! Why won't you die?... Why won't you die?")
exit(1)
for i in idx_ptrack:
@ -60,7 +61,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack has correct bits after recovery
self.check_ptrack_recovery(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

21
tests/ptrack_vacuum.py
View File

@ -1,8 +1,8 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import unittest
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,9 +10,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -78,7 +79,5 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

18
tests/ptrack_vacuum_bits_frozen.py
View File

@ -1,8 +1,8 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,9 +10,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_bits_frozen'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -69,7 +70,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

18
tests/ptrack_vacuum_bits_visibility.py
View File

@ -1,8 +1,8 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,9 +10,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_bits_visibility'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -69,7 +70,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

16
tests/ptrack_vacuum_full.py
View File

@ -1,9 +1,8 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
#import os
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -11,10 +10,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_full'
def teardown(self):
# clean_all()
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_ptrack_vacuum_full(self):
@ -72,7 +67,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity, the most important part
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

18
tests/ptrack_vacuum_truncate.py
View File

@ -1,8 +1,8 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
from testgres import stop_all, clean_all
from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack
import shutil
class SimpleTest(ProbackupTest, unittest.TestCase):
@ -10,9 +10,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_truncate'
def teardown(self):
# clean_all()
stop_all()
# @classmethod
# def tearDownClass(self):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -71,7 +72,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
node.stop()
if __name__ == '__main__':
unittest.main()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

26
tests/replica.py
View File

@ -1,13 +1,12 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack
from datetime import datetime, timedelta
from testgres import stop_all
from testgres import stop_all, clean_all
import subprocess
from sys import exit, _getframe
import shutil
from sys import exit
import time
import shutil
class ReplicaTest(ProbackupTest, unittest.TestCase):
@ -16,9 +15,10 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
super(ReplicaTest, self).__init__(*args, **kwargs)
self.module_name = 'replica'
# @classmethod
# def tearDownClass(cls):
# stop_all()
# @classmethod
# def tearDownClass(cls):
# clean_all()
# shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True)
# @unittest.skip("skip")
# @unittest.expectedFailure
@ -65,16 +65,18 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# Make backup from replica
self.add_instance(backup_dir, 'slave', slave)
#time.sleep(2)
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose',
'--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, 'slave')
self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status'])
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_replica_archive_full_backup(self):
"""make full archive backup from replica"""
"""make full archive backup from replica, set replica, make backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
@ -127,3 +129,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300',
'--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])
self.validate_pb(backup_dir, 'slave')
self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status'])
# Clean after yourself
self.del_test_dir(self.module_name, fname)

193
tests/restore_test.py
View File

@ -1,11 +1,9 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all, clean_all
import subprocess
from datetime import datetime
import shutil
from sys import exit
@ -15,10 +13,6 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
super(RestoreTest, self).__init__(*args, **kwargs)
self.module_name = 'restore'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_restore_full_to_latest(self):
@ -45,7 +39,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.cleanup()
# 1 - Test recovery from latest
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -58,7 +52,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_full_page_to_latest(self):
@ -89,7 +84,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -98,10 +93,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_to_timeline(self):
#@unittest.skip("skip")
def test_restore_to_specific_timeline(self):
"""recovery to target timeline"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
@ -120,11 +116,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(backup_dir, 'node', node)
target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")])
target_tli = int(node.get_control_data()["Latest checkpoint's TimeLineID"])
node.stop({"-m": "immediate"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -134,13 +130,13 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
pgbench.stdout.close()
self.backup_node(backup_dir, 'node', node, backup_type="full")
self.backup_node(backup_dir, 'node', node)
node.stop({"-m": "immediate"})
node.cleanup()
# Correct Backup must be choosen for restore
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -152,7 +148,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_to_time(self):
@ -181,7 +178,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -190,7 +187,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_to_xid(self):
@ -235,7 +233,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "fast"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -244,7 +242,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_full_ptrack_archive(self):
@ -275,7 +274,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -284,7 +283,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_ptrack(self):
@ -321,7 +321,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -330,7 +330,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_full_ptrack_stream(self):
@ -362,7 +363,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -371,7 +372,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
after = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.assertEqual(before, after)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_full_ptrack_under_load(self):
@ -410,7 +412,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -418,10 +420,10 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
self.assertEqual(bbalance, delta)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_full_under_load_ptrack(self):
@ -463,16 +465,17 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.cleanup()
#self.wrong_wal_clean(node, wal_segment_size)
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
self.assertEqual(bbalance, delta)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_to_xid_inclusive(self):
@ -501,9 +504,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
with node.connect("postgres") as con:
res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
con.commit()
target_xid = res[0][0]
target_xid = result[0][0]
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
@ -517,7 +520,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "fast"})
node.cleanup()
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -528,7 +531,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.assertEqual(before, after)
self.assertEqual(len(node.execute("postgres", "SELECT * FROM tbl0005")), 0)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_with_tablespace_mapping_1(self):
@ -556,7 +560,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
con.commit()
backup_id = self.backup_node(backup_dir, 'node', node)
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
# 1 - Try to restore to existing directory
node.stop()
@ -565,7 +569,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -577,20 +581,20 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message,
'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# 3 - Restore using tablespace-mapping
tblspc_path_new = os.path.join(node.base_dir, "tblspc_new")
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
res = node.execute("postgres", "SELECT id FROM test")
self.assertEqual(res[0][0], 1)
result = node.execute("postgres", "SELECT id FROM test")
self.assertEqual(result[0][0], 1)
# 4 - Restore using tablespace-mapping using page backup
self.backup_node(backup_dir, 'node', node)
@ -600,22 +604,23 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
show_pb = self.show_pb(backup_dir, 'node')
self.assertEqual(show_pb[1]['Status'], six.b("OK"))
self.assertEqual(show_pb[2]['Status'], six.b("OK"))
self.assertEqual(show_pb[1]['Status'], "OK")
self.assertEqual(show_pb[2]['Status'], "OK")
node.stop()
node.cleanup()
tblspc_path_page = os.path.join(node.base_dir, "tblspc_page")
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
res = node.execute("postgres", "SELECT id FROM test OFFSET 1")
self.assertEqual(res[0][0], 2)
result = node.execute("postgres", "SELECT id FROM test OFFSET 1")
self.assertEqual(result[0][0], 2)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_restore_with_tablespace_mapping_2(self):
@ -633,7 +638,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# Full backup
self.backup_node(backup_dir, 'node', node)
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK")
# Create tablespace
tblspc_path = os.path.join(node.base_dir, "tblspc")
@ -647,8 +652,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# First page backup
self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE"))
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK")
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE")
# Create tablespace table
with node.connect("postgres") as con:
@ -661,15 +666,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# Second page backup
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE"))
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK")
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE")
node.stop()
node.cleanup()
tblspc_path_new = os.path.join(node.base_dir, "tblspc_new")
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
@ -678,9 +683,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.assertEqual(count[0][0], 4)
count = node.execute("postgres", "SELECT count(*) FROM tbl1")
self.assertEqual(count[0][0], 4)
node.stop()
# @unittest.skip("skip")
# Clean after yourself
self.del_test_dir(self.module_name, fname)
#@unittest.skip("skip")
def test_archive_node_backup_stream_restore_to_recovery_time(self):
"""make node with archiving, make stream backup, make PITR to Recovery Time"""
fname = self.id().split('.')[3]
@ -692,6 +699,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"])
@ -700,18 +708,57 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
node.cleanup()
recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time']
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
res = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in res[2])
result = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in result[2].decode("utf-8"))
self.assertEqual(True, node.status())
# Clean after yourself
self.del_test_dir(self.module_name, fname)
#@unittest.skip("skip")
def test_archive_node_backup_stream_restore_to_recovery_time(self):
"""make node with archiving, make stream backup, make PITR to Recovery Time"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"])
node.psql("postgres", "create table t_heap(a int)")
node.psql("postgres", "select pg_switch_xlog()")
node.stop()
node.cleanup()
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
result = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in result[2].decode("utf-8"))
self.assertEqual(True, node.status())
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_archive_node_backup_stream_pitr(self):
@ -734,16 +781,18 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time']
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
res = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in res[2])
node.stop()
result = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in result[2].decode("utf-8"))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_archive_node_backup_archive_pitr_2(self):
@ -766,13 +815,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
res = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in res[2])
node.stop()
result = node.psql("postgres", 'select * from t_heap')
self.assertTrue('does not exist' in result[2].decode("utf-8"))
# Clean after yourself
self.del_test_dir(self.module_name, fname)

25
tests/retention_test.py
View File

@ -2,8 +2,9 @@ import unittest
import os
from datetime import datetime, timedelta
from os import path, listdir
from helpers.ptrack_helpers import ProbackupTest
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest
from testgres import stop_all, clean_all
import shutil
class RetentionTest(ProbackupTest, unittest.TestCase):
@ -12,10 +13,6 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
super(RetentionTest, self).__init__(*args, **kwargs)
self.module_name = 'retention'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_retention_redundancy_1(self):
@ -51,17 +48,18 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
min_wal = None
max_wal = None
for line in log.splitlines():
if line.startswith(b"INFO: removed min WAL segment"):
if line.startswith("INFO: removed min WAL segment"):
min_wal = line[31:-1]
elif line.startswith(b"INFO: removed max WAL segment"):
elif line.startswith("INFO: removed max WAL segment"):
max_wal = line[31:-1]
for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')):
if not wal_name.endswith(".backup"):
wal_name_b = wal_name.encode('ascii')
self.assertEqual(wal_name_b[8:] > min_wal[8:], True)
self.assertEqual(wal_name_b[8:] > max_wal[8:], True)
#wal_name_b = wal_name.encode('ascii')
self.assertEqual(wal_name[8:] > min_wal[8:], True)
self.assertEqual(wal_name[8:] > max_wal[8:], True)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("123")
def test_retention_window_2(self):
@ -106,4 +104,5 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.delete_expired(backup_dir, 'node')
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)

24
tests/show_test.py
View File

@ -1,9 +1,9 @@
import unittest
import os
from os import path
import six
from helpers.ptrack_helpers import ProbackupTest
from testgres import stop_all
from .helpers.ptrack_helpers import ProbackupTest
from testgres import stop_all, clean_all
import shutil
class OptionTest(ProbackupTest, unittest.TestCase):
@ -12,10 +12,6 @@ class OptionTest(ProbackupTest, unittest.TestCase):
super(OptionTest, self).__init__(*args, **kwargs)
self.module_name = 'show'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_show_1(self):
@ -36,8 +32,10 @@ class OptionTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]),
None
)
self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True))
node.stop()
self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_corrupt_2(self):
@ -57,9 +55,11 @@ class OptionTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(backup_dir, 'node', node)
# delete file which belong to backup
file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf")
file = path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf")
os.remove(file)
self.validate_pb(backup_dir, 'node', backup_id)
self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True))
node.stop()
self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True))
# Clean after yourself
self.del_test_dir(self.module_name, fname)

75
tests/validate_test.py
View File

@ -1,12 +1,10 @@
import unittest
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from datetime import datetime, timedelta
from testgres import stop_all
from testgres import stop_all, clean_all
import subprocess
from sys import exit
import re
class ValidateTest(ProbackupTest, unittest.TestCase):
@ -15,10 +13,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
super(ValidateTest, self).__init__(*args, **kwargs)
self.module_name = 'validate'
@classmethod
def tearDownClass(cls):
stop_all()
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_validate_wal_unreal_values(self):
@ -55,7 +49,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
after_backup_time = datetime.now().replace(second=0, microsecond=0)
# Validate to real time
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.assertIn("INFO: backup validation completed successfully",
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -65,7 +59,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -75,7 +69,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertTrue('ERROR: not enough WAL records to time' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -87,7 +81,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
target_xid = res[0][0]
node.execute("postgres", "SELECT pg_switch_xlog()")
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.assertIn("INFO: backup validation completed successfully",
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
@ -97,15 +91,18 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertTrue('ERROR: not enough WAL records to xid' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Validate with backup ID
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.assertIn("INFO: backup validation completed successfully",
self.validate_pb(backup_dir, 'node', backup_id),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_validate_corrupt_wal_1(self):
"""make archive node, make archive backup, corrupt all wal files, run validate, expect errors"""
@ -131,22 +128,25 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
f = open(os.path.join(wals_dir, wal), "rb+")
f.seek(42)
f.write(six.b("blablablaadssaaaaaaaaaaaaaaa"))
f.close
with open(os.path.join(wals_dir, wal), "rb+", 0) as f:
f.seek(42)
f.write(b"blablablaadssaaaaaaaaaaaaaaa")
f.flush()
f.close
# Simple validate
try:
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_validate_corrupt_wal_2(self):
@ -178,22 +178,25 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
f = open(os.path.join(wals_dir, wal), "rb+")
f.seek(0)
f.write(six.b("blablabla"))
f.close
with open(os.path.join(wals_dir, wal), "rb+", 0) as f:
f.seek(0)
f.write(b"blablablaadssaaaaaaaaaaaaaaa")
f.flush()
f.close
# Validate to xid
try:
self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)])
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_validate_wal_lost_segment_1(self):
@ -232,7 +235,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format(
file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
@ -243,10 +246,12 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
except ProbackupException as e:
self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)
# @unittest.skip("skip")
def test_validate_wal_lost_segment_2(self):
@ -291,12 +296,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
self.output, self.cmd))
except ProbackupException, e:
self.assertTrue('INFO: wait for LSN'
and 'in archived WAL segment'
and 'WARNING: could not read WAL record at'
except ProbackupException as e:
self.assertTrue('INFO: wait for LSN' in e.message
and 'in archived WAL segment' in e.message
and 'WARNING: could not read WAL record at' in e.message
and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"')
node.stop()
# Clean after yourself
self.del_test_dir(self.module_name, fname)