db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-08 14:28:36 +02:00
Code Issues Releases Activity

tests fix

Browse Source
This commit is contained in:
Grigory Smolkin 2017-06-20 13:57:23 +03:00
parent 5e034a4acc
commit 1a849cad7e
23 changed files with 1029 additions and 754 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
tests/__init__.py
View File

@ -11,28 +11,28 @@ from . import init_test, option_test, show_test, \
def load_tests(loader, tests, pattern):
suite = unittest.TestSuite()
# suite.addTests(loader.loadTestsFromModule(replica))
suite.addTests(loader.loadTestsFromModule(replica))
# suite.addTests(loader.loadTestsFromModule(pgpro560))
# suite.addTests(loader.loadTestsFromModule(pgpro589))
suite.addTests(loader.loadTestsFromModule(pgpro688))
# suite.addTests(loader.loadTestsFromModule(pgpro688))
# suite.addTests(loader.loadTestsFromModule(false_positive))
# suite.addTests(loader.loadTestsFromModule(init_test))
# suite.addTests(loader.loadTestsFromModule(option_test))
# suite.addTests(loader.loadTestsFromModule(show_test))
# suite.addTests(loader.loadTestsFromModule(backup_test))
# suite.addTests(loader.loadTestsFromModule(delete_test))
# suite.addTests(loader.loadTestsFromModule(restore_test))
# suite.addTests(loader.loadTestsFromModule(validate_test))
# suite.addTests(loader.loadTestsFromModule(retention_test))
# suite.addTests(loader.loadTestsFromModule(ptrack_clean))
# suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
# suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))
# suite.addTests(loader.loadTestsFromModule(ptrack_recovery))
# suite.addTests(loader.loadTestsFromModule(ptrack_vacuum))
# suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen))
# suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility))
# suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full))
# suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate))
suite.addTests(loader.loadTestsFromModule(init_test))
suite.addTests(loader.loadTestsFromModule(option_test))
suite.addTests(loader.loadTestsFromModule(show_test))
suite.addTests(loader.loadTestsFromModule(backup_test))
suite.addTests(loader.loadTestsFromModule(delete_test))
suite.addTests(loader.loadTestsFromModule(restore_test))
suite.addTests(loader.loadTestsFromModule(validate_test))
suite.addTests(loader.loadTestsFromModule(retention_test))
suite.addTests(loader.loadTestsFromModule(ptrack_clean))
suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))
suite.addTests(loader.loadTestsFromModule(ptrack_recovery))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full))
suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate))
return suite

176
tests/backup_test.py
View File

@ -1,6 +1,7 @@
import unittest
from os import path, listdir
import os
import six
from time import sleep
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
@ -9,6 +10,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(BackupTest, self).__init__(*args, **kwargs)
self.module_name = 'backup'
@classmethod
def tearDownClass(cls):
@ -20,132 +22,198 @@ class BackupTest(ProbackupTest, unittest.TestCase):
def test_backup_modes_archive(self):
"""standart backup modes with ARCHIVE WAL method"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
# full backup mode
#with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log:
# backup_log.write(self.backup_pb(node, options=["--verbose"]))
# backup_log.write(self.backup_node(node, options=["--verbose"]))
self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
show_backup = self.show_pb(backup_dir, 'node')[0]
show_backup = self.show_pb(node)[0]
full_backup_id = show_backup['ID']
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("FULL"))
# postmaster.pid and postmaster.opts shouldn't be copied
excluded = True
backups_dir = path.join(self.backup_dir(node), "backups")
for backup in listdir(backups_dir):
db_dir = path.join(backups_dir, backup, "database")
for f in listdir(db_dir):
if path.isfile(path.join(db_dir, f)) and \
(f == "postmaster.pid" or f == "postmaster.opts"):
db_dir = os.path.join(backup_dir, "backups", 'node', backup_id, "database")
for f in os.listdir(db_dir):
if os.path.isfile(os.path.join(db_dir, f)) \
and (f == "postmaster.pid" or f == "postmaster.opts"):
excluded = False
self.assertEqual(excluded, True)
# page backup mode
self.backup_pb(node, backup_type="page")
page_backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
# print self.show_pb(node)
show_backup = self.show_pb(node)[1]
show_backup = self.show_pb(backup_dir, 'node')[1]
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("PAGE"))
# Check parent backup
self.assertEqual(
full_backup_id,
self.show_pb(node, id=show_backup['ID'])["parent-backup-id"])
backup_id,
self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"])
# ptrack backup mode
self.backup_pb(node, backup_type="ptrack")
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
show_backup = self.show_pb(node)[2]
show_backup = self.show_pb(backup_dir, 'node')[2]
self.assertEqual(show_backup['Status'], six.b("OK"))
self.assertEqual(show_backup['Mode'], six.b("PTRACK"))
# Check parent backup
self.assertEqual(
page_backup_id,
self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"])
node.stop()
# @unittest.skip("skip")
def test_smooth_checkpoint(self):
"""full backup with smooth checkpoint"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, options=["-C"])
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
self.backup_node(backup_dir, 'node' ,node, options=["-C"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
node.stop()
def test_page_backup_without_full(self):
#@unittest.skip("skip")
def test_incremental_backup_without_full(self):
"""page-level backup without validated full backup"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
try:
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node, backup_type="page")
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
pass
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("ERROR"))
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
sleep(1)
try:
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR"))
node.stop()
@unittest.expectedFailure
# Need to forcibly validate parent
def test_incremental_backup_corrupt_full(self):
"""page-level backup with corrupted full backup"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
backup_id = self.backup_node(backup_dir, 'node', node)
file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf")
os.remove(file)
try:
self.backup_node(backup_dir, 'node', node, backup_type="page")
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
sleep(1)
self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR"))
node.stop()
# @unittest.skip("skip")
def test_ptrack_threads(self):
"""ptrack multi thread backup mode"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', "ptrack_enable": "on", 'max_wal_senders': '2'}
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, backup_type="full", options=["-j", "4"])
self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log:
backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["-j", "4"]))
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"])
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
node.stop()
# @unittest.skip("skip")
def test_ptrack_threads_stream(self):
"""ptrack multi thread backup mode and stream"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, backup_type="full", options=["-j", "4", "--stream"])
self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"])
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
self.backup_pb(node, backup_type="ptrack", options=["-j", "4", "--stream"])
self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"])
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK"))
node.stop()

93
tests/delete_test.py
View File

@ -1,5 +1,5 @@
import unittest
from os import path
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
@ -10,6 +10,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(DeleteTest, self).__init__(*args, **kwargs)
self.module_name = 'delete'
@classmethod
def tearDownClass(cls):
@ -20,71 +21,109 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
def test_delete_full_backups(self):
"""delete full backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init()
# full backup mode
self.backup_pb(node)
# full backup
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
show_backups = self.show_pb(node)
show_backups = self.show_pb(backup_dir, 'node')
id_1 = show_backups[0]['ID']
id_2 = show_backups[1]['ID']
id_3 = show_backups[2]['ID']
self.delete_pb(node, show_backups[1]['ID'])
show_backups = self.show_pb(node)
self.delete_pb(backup_dir, 'node', id_2)
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(show_backups[0]['ID'], id_1)
self.assertEqual(show_backups[1]['ID'], id_3)
node.stop()
def test_delete_increment(self):
def test_delete_increment_page(self):
"""delete increment and all after him"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
# full backup mode
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
# page backup mode
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node, backup_type="page")
# page backup mode
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node, backup_type="page")
# full backup mode
self.backup_pb(node)
show_backups = self.show_pb(node)
self.backup_node(backup_dir, 'node', node)
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 4)
# delete first page backup
self.delete_pb(node, show_backups[1]['ID'])
self.delete_pb(backup_dir, 'node', show_backups[1]['ID'])
show_backups = self.show_pb(node)
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 2)
self.assertEqual(show_backups[0]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[0]['Status'], six.b("OK"))
self.assertEqual(show_backups[1]['Mode'], six.b("FULL"))
self.assertEqual(show_backups[1]['Status'], six.b("OK"))
node.stop()
def test_delete_increment_ptrack(self):
"""delete increment and all after him"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
# full backup mode
self.backup_node(backup_dir, 'node', node)
# page backup mode
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
# page backup mode
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
# full backup mode
self.backup_node(backup_dir, 'node', node)
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 4)
# delete first page backup
self.delete_pb(backup_dir, 'node', show_backups[1]['ID'])
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(len(show_backups), 2)
self.assertEqual(show_backups[0]['Mode'], six.b("FULL"))

50
tests/expected/option_help.out
View File

@ -5,10 +5,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
pg_probackup version
pg_probackup init -B backup-path -D pgdata-dir
pg_probackup init -B backup-path
pg_probackup set-config -B backup-dir
[-d dbname] [-h host] [-p port] [-U username]
pg_probackup set-config -B backup-dir --instance=instance_name
[--log-level=log-level]
[--log-filename=log-filename]
[--error-log-filename=error-log-filename]
@ -17,36 +16,47 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[--log-rotation-age=log-rotation-age]
[--retention-redundancy=retention-redundancy]
[--retention-window=retention-window]
[--compress-algorithm=compress-algorithm]
[--compress-level=compress-level]
[-d dbname] [-h host] [-p port] [-U username]
[--master-db=db_name] [--master-host=host_name]
[--master-port=port] [--master-user=user_name]
[--replica-timeout=timeout]
pg_probackup show-config -B backup-dir
pg_probackup show-config -B backup-dir --instance=instance_name
pg_probackup backup -B backup-path -b backup-mode
[-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log]
pg_probackup backup -B backup-path -b backup-mode --instance=instance_name
[-C] [--stream [-S slot-name]] [--backup-pg-log]
[-j num-threads] [--archive-timeout=archive-timeout]
[--compress-algorithm=compress-algorithm]
[--compress-level=compress-level]
[--progress] [--delete-expired]
[-d dbname] [-h host] [-p port] [-U username]
[--master-db=db_name] [--master-host=host_name]
[--master-port=port] [--master-user=user_name]
[--replica-timeout=timeout]
pg_probackup restore -B backup-dir
[-D pgdata-dir] [-i backup-id] [--progress]
[--time=time|--xid=xid [--inclusive=boolean]]
[--timeline=timeline] [-T OLDDIR=NEWDIR]
pg_probackup restore -B backup-dir --instance=instance_name
[-D pgdata-dir] [-i backup-id] [--progress]
[--time=time|--xid=xid [--inclusive=boolean]]
[--timeline=timeline] [-T OLDDIR=NEWDIR]
pg_probackup validate -B backup-dir
[-D pgdata-dir] [-i backup-id] [--progress]
[--time=time|--xid=xid [--inclusive=boolean]]
[--timeline=timeline]
pg_probackup validate -B backup-dir [--instance=instance_name]
[-i backup-id] [--progress]
[--time=time|--xid=xid [--inclusive=boolean]]
[--timeline=timeline]
pg_probackup show -B backup-dir
[-i backup-id]
[--instance=instance_name [-i backup-id]]
pg_probackup delete -B backup-dir
[--wal] [-i backup-id | --expired]
pg_probackup delete -B backup-dir --instance=instance_name
[--wal] [-i backup-id | --expired]
pg_probackup add-instance -B backup-dir
--instance=instance_name
pg_probackup add-instance -B backup-dir -D pgdata-dir
--instance=instance_name
pg_probackup del-instance -B backup-dir
--instance=instance_name
--instance=instance_name
Read the website for details. <https://github.com/postgrespro/pg_probackup>
Report bugs to <https://github.com/postgrespro/pg_probackup/issues>.

1
tests/false_positive.py
View File

@ -12,6 +12,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(FalsePositive, self).__init__(*args, **kwargs)
self.module_name = 'false_positive'
@classmethod
def tearDownClass(cls):

45
tests/helpers/ptrack_helpers.py
View File

@ -7,6 +7,7 @@ import six
from testgres import get_new_node
import hashlib
import re
import pwd
idx_ptrack = {
@ -143,6 +144,7 @@ class ProbackupTest(object):
pass
self.probackup_path = os.path.abspath(os.path.join(
self.dir_path, "../pg_probackup"))
self.user = self.get_username()
def arcwal_dir(self, node):
return "%s/backup/wal" % node.base_dir
@ -329,6 +331,7 @@ class ProbackupTest(object):
def init_pb(self, backup_dir):
shutil.rmtree(backup_dir, ignore_errors=True)
return self.run_pb([
"init",
"-B", backup_dir
@ -352,8 +355,8 @@ class ProbackupTest(object):
"-D", node.data_dir
])
def clean_pb(self, node):
shutil.rmtree(self.backup_dir(node), ignore_errors=True)
def clean_pb(self, backup_dir):
shutil.rmtree(backup_dir, ignore_errors=True)
def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False):
@ -370,11 +373,9 @@ class ProbackupTest(object):
return self.run_pb(cmd_list + options, async)
def restore_node(self, backup_dir, instance, data_dir=None, id=None, options=[]):
def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]):
if data_dir is None:
data_dir = node.data_dir
if backup_dir is None:
backup_dir = self.backup_dir(node)
cmd_list = [
"restore",
@ -382,8 +383,8 @@ class ProbackupTest(object):
"-D", data_dir,
"--instance={0}".format(instance)
]
if id:
cmd_list += ["-i", id]
if backup_id:
cmd_list += ["-i", backup_id]
return self.run_pb(cmd_list + options)
@ -449,44 +450,46 @@ class ProbackupTest(object):
specific_record[name.strip()] = var
return specific_record
def validate_pb(self, backup_dir, instance=None, id=None, options=[]):
def validate_pb(self, backup_dir, instance=None, backup_id=None, options=[]):
cmd_list = [
"validate",
"-B", backup_dir,
"-B", backup_dir
]
if instance:
cmd_list += ["--instance={0}".format(instance)]
if id:
cmd_list += ["-i", id]
if backup_id:
cmd_list += ["-i", backup_id]
return self.run_pb(cmd_list + options)
def delete_pb(self, backup_dir, instance=None, id=None, options=[]):
def delete_pb(self, backup_dir, instance=None, backup_id=None, options=[]):
cmd_list = [
"delete",
"-B", self.backup_dir(node),
"-B", backup_dir
]
if instance:
cmd_list += ["--instance={0}".format(instance)]
if id:
cmd_list += ["-i", id]
if backup_id:
cmd_list += ["-i", backup_id]
# print(cmd_list)
return self.run_pb(cmd_list + options)
def delete_expired(self, backup_dir, instance=None, options=[]):
def delete_expired(self, backup_dir, instance, options=[]):
cmd_list = [
"delete", "--expired",
"-B", self.backup_dir(node),
"-B", backup_dir,
"--instance={0}".format(instance)
]
return self.run_pb(cmd_list + options)
def show_config(self, backup_dir, instance=None):
def show_config(self, backup_dir, instance):
out_dict = {}
cmd_list = [
"show-config",
"-B", self.backup_dir(node),
"-B", backup_dir,
"--instance={0}".format(instance)
]
res = self.run_pb(cmd_list).splitlines()
for line in res:
@ -556,3 +559,7 @@ class ProbackupTest(object):
return str(var[0][0])
else:
return False
def get_username(self):
""" Returns current user name """
return pwd.getpwuid(os.getuid())[0]

30
tests/init_test.py
View File

@ -10,51 +10,55 @@ class InitTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(InitTest, self).__init__(*args, **kwargs)
self.module_name = 'init'
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_success(self):
"""Success normal init"""
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
self.assertEqual(self.init_pb(node), six.b(""))
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname))
self.init_pb(backup_dir)
self.assertEqual(
dir_files(self.backup_dir(node)),
dir_files(backup_dir),
['backups', 'wal']
)
self.add_instance(node=node, instance='test')
self.add_instance(backup_dir, 'node', node)
self.assertEqual("INFO: Instance 'test' deleted successfully\n",
self.del_instance(node=node, instance='test'),
self.assertEqual("INFO: Instance 'node' successfully deleted\n",
self.del_instance(backup_dir, 'node', node),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
try:
self.show_pb(node, instance='test')
self.show_pb(backup_dir, 'node')
self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
"ERROR: Instance 'test' does not exist in this backup catalog\n",
"ERROR: Instance 'node' does not exist in this backup catalog\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd))
def test_already_exist(self):
"""Failure with backup catalog already existed"""
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
self.init_pb(node)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname))
self.init_pb(backup_dir)
try:
self.init_pb(node)
self.show_pb(backup_dir, 'node')
self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
"ERROR: backup catalog already exist and it's not empty\n",
"ERROR: Instance 'node' does not exist in this backup catalog\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
def test_abs_path(self):
"""failure with backup catalog should be given as absolute path"""
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname))
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname))
try:
self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)])
self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format(

188
tests/option_test.py
View File

@ -1,5 +1,5 @@
import unittest
from os import path
import os
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
@ -9,6 +9,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(OptionTest, self).__init__(*args, **kwargs)
self.module_name = 'option'
@classmethod
def tearDownClass(cls):
@ -19,166 +20,195 @@ class OptionTest(ProbackupTest, unittest.TestCase):
def test_help_1(self):
"""help options"""
fname = self.id().split(".")[3]
with open(path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out:
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out:
self.assertEqual(
self.run_pb(["--help"]),
help_out.read()
)
# @unittest.skip("skip")
def test_version_2(self):
"""help options"""
fname = self.id().split(".")[3]
with open(path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out:
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out:
self.assertEqual(
self.run_pb(["--version"]),
version_out.read()
)
# @unittest.skip("skip")
def test_without_backup_path_3(self):
"""backup command failure without backup mode option"""
fname = self.id().split(".")[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
try:
self.run_pb(["backup", "-b", "full"])
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n'
)
self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# @unittest.skip("skip")
def test_options_4(self):
"""check options test"""
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/option/{0}".format(fname))
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'})
try:
node.stop()
except:
pass
self.assertEqual(self.init_pb(node), six.b(""))
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
# backup command failure without instance option
try:
self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"])
self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(e.message,
'ERROR: required parameter not specified: --instance\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# backup command failure without backup mode option
try:
self.run_pb(["backup", "-B", self.backup_dir(node), "-D", node.data_dir])
# we should die here because exception is what we expect to happen
exit(1)
self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir])
self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
# print e.message
self.assertEqual(
e.message,
'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n'
)
self.assertEqual(e.message,
'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# backup command failure with invalid backup mode option
try:
self.run_pb(["backup", "-b", "bad", "-B", self.backup_dir(node)])
# we should die here because exception is what we expect to happen
exit(1)
self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"])
self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: invalid backup-mode "bad"\n'
)
self.assertEqual(e.message,
'ERROR: invalid backup-mode "bad"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# delete failure without ID
try:
self.run_pb(["delete", "-B", self.backup_dir(node)])
self.run_pb(["delete", "-B", backup_dir, "--instance=node"])
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: required backup ID not specified\n'
)
self.assertEqual(e.message,
'ERROR: required backup ID not specified\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
#@unittest.skip("skip")
def test_options_5(self):
"""check options test"""
fname = self.id().split(".")[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'})
self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir)))
self.add_instance(backup_dir, 'node', node)
node.start()
# syntax error in pg_probackup.conf
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write(" = INFINITE\n")
try:
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: syntax error in " = INFINITE"\n'
)
self.assertEqual(e.message,
'ERROR: syntax error in " = INFINITE"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.clean_pb(node)
self.assertEqual(self.init_pb(node), six.b(""))
self.clean_pb(backup_dir)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
# invalid value in pg_probackup.conf
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write("BACKUP_MODE=\n")
try:
self.backup_pb(node, backup_type=None),
self.backup_node(backup_dir, 'node', node, backup_type=None),
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: invalid backup-mode ""\n'
)
self.assertEqual(e.message,
'ERROR: invalid backup-mode ""\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.clean_pb(node)
self.clean_pb(backup_dir)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
# Command line parameters should override file values
self.assertEqual(self.init_pb(node), six.b(""))
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write("retention-redundancy=1\n")
self.assertEqual(
self.show_config(node)['retention-redundancy'],
self.show_config(backup_dir, 'node')['retention-redundancy'],
six.b('1')
)
# User cannot send --system-identifier parameter via command line
try:
self.backup_pb(node, options=["--system-identifier", "123"]),
self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]),
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: option system-identifier cannot be specified in command line\n'
)
self.assertEqual(e.message,
'ERROR: option system-identifier cannot be specified in command line\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# invalid value in pg_probackup.conf
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write("SMOOTH_CHECKPOINT=FOO\n")
try:
self.backup_pb(node),
self.backup_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
"ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n"
)
self.assertEqual(e.message,
"ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n",
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.clean_pb(node)
self.assertEqual(self.init_pb(node), six.b(""))
self.clean_pb(backup_dir)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
# invalid option in pg_probackup.conf
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf:
conf.write("TIMELINEID=1\n")
try:
self.backup_pb(node),
self.backup_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
exit(1)
self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: invalid option "TIMELINEID"\n'
)
self.assertEqual(e.message,
'ERROR: invalid option "TIMELINEID"\n',
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.clean_pb(node)
self.assertEqual(self.init_pb(node), six.b(""))
node.stop()
# self.clean_pb(backup_dir)
# node.stop()

166
tests/pgpro688.py
View File

@ -15,19 +15,17 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ReplicaTest, self).__init__(*args, **kwargs)
self.module_name = 'replica'
self.instance_1 = 'master'
self.instance_2 = 'slave'
self.instance_master = 'master'
self.instance_replica = 'replica'
# @classmethod
# def tearDownClass(cls):
# stop_all()
# @unittest.skip("skip")
@unittest.skip("skip")
# @unittest.expectedFailure
def test_replica_stream_full_backup(self):
"""
make full stream backup from replica
"""
"""make full stream backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
@ -35,59 +33,127 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'}
)
master.start()
shutil.rmtree(backup_dir, ignore_errors=True)
self.init_pb(backup_dir)
instance = ''
self.add_instance(backup_dir, self.instance_1, master)
self.set_archiving(backup_dir, self.instance_1, master)
master.restart()
self.add_instance(backup_dir, self.instance_master, master)
master.start()
slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname))
slave_port = slave.port
slave.cleanup()
# Make empty Object 'replica' from new node
replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname))
replica_port = replica.port
replica.cleanup()
# FULL BACKUP
self.backup_node(backup_dir, self.instance_1, master, backup_type='full', options=['--stream'])
# FULL STREAM backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream'])
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
#PAGE BACKUP
self.backup_node(backup_dir, self.instance_1, master, backup_type='page', options=['--stream'])
self.restore_node(backup_dir, self.instance_1, slave.data_dir)
# FULL STREAM backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream'])
slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port))
slave.append_conf('postgresql.auto.conf', 'hot_standby = on')
slave.append_conf('recovery.conf', "standby_mode = 'on'")
slave.append_conf('recovery.conf',
# Restore last backup from master to Replica directory
self.restore_node(backup_dir, self.instance_master, replica.data_dir)
# Set Replica
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port))
slave.start({"-t": "600"})
# Replica Ready
replica.start({"-t": "600"})
# Check replica
after = slave.execute("postgres", "SELECT * FROM t_heap")
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make backup from replica
self.add_instance(backup_dir, self.instance_2, slave)
# Add instance replica
self.add_instance(backup_dir, self.instance_replica, replica)
# FULL STREAM backup of replica
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_2, slave, backup_type='full', options=[
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[
'--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, self.instance_2)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_2)[0]['Status'])
# Validate instance replica
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
def test_replica_archive_full_backup(self):
"""make page archive backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'}
)
self.set_archiving(backup_dir, self.instance_master, master)
self.init_pb(backup_dir)
self.add_instance(backup_dir, self.instance_master, master)
master.start()
# Make empty Object 'replica' from new node
replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname))
replica_port = replica.port
replica.cleanup()
# FULL ARCHIVE backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='full')
# Create table t_heap
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
# PAGE ARCHIVE backup of master
self.backup_node(backup_dir, self.instance_master, master, backup_type='page')
# Restore last backup from master to Replica directory
self.restore_node(backup_dir, self.instance_master, replica.data_dir)
# Set Replica
self.set_archiving(backup_dir, self.instance_replica, replica, replica=True)
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port))
replica.start({"-t": "600"})
# Check replica
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make FULL ARCHIVE backup from replica
self.add_instance(backup_dir, self.instance_replica, replica)
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[
'--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
# Drop Table t_heap
after = master.execute("postgres", "drop table t_heap")
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
# Make page backup from replica
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[
'--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, self.instance_replica)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status'])
@unittest.skip("skip")
def test_replica_archive_full_backup(self):
def test_replica_archive_full_backup_123(self):
"""
make full archive backup from replica
"""
fname = self.id().split('.')[3]
master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname),
set_archiving=True,
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
@ -95,9 +161,9 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
master.append_conf('postgresql.auto.conf', 'archive_timeout = 10')
master.start()
slave = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/slave".format(fname))
slave_port = slave.port
slave.cleanup()
replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname))
replica_port = replica.port
replica.cleanup()
self.assertEqual(self.init_pb(master), six.b(""))
self.backup_pb(node=master, backup_type='full', options=['--stream'])
@ -109,27 +175,27 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
before = master.execute("postgres", "SELECT * FROM t_heap")
id = self.backup_pb(master, backup_type='page', options=['--stream'])
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=slave.data_dir)
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir)
# Settings for Replica
slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port))
slave.append_conf('postgresql.auto.conf', 'hot_standby = on')
replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port))
replica.append_conf('postgresql.auto.conf', 'hot_standby = on')
# Set Archiving for replica
self.set_archiving_conf(slave, replica=True)
self.set_archiving_conf(replica, replica=True)
slave.append_conf('recovery.conf', "standby_mode = 'on'")
slave.append_conf('recovery.conf',
replica.append_conf('recovery.conf', "standby_mode = 'on'")
replica.append_conf('recovery.conf',
"primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port))
slave.start({"-t": "600"})
replica.start({"-t": "600"})
# Replica Started
# master.execute("postgres", "checkpoint")
# Check replica
after = slave.execute("postgres", "SELECT * FROM t_heap")
after = replica.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make backup from replica
self.assertEqual(self.init_pb(slave), six.b(""))
self.backup_pb(slave, backup_type='full', options=['--archive-timeout=30'])
self.validate_pb(slave)
self.assertEqual(self.init_pb(replica), six.b(""))
self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30'])
self.validate_pb(replica)

23
tests/ptrack_clean.py
View File

@ -1,4 +1,5 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_clean'
def teardown(self):
stop_all()
@ -15,13 +17,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_clean(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
set_archiving=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -35,8 +40,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
# Make full backup to clean every ptrack
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
for i in idx_ptrack:
# get fork size and calculate it in pages
@ -52,7 +56,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
node.psql('postgres', 'update t_heap set text = md5(text), tsvector = md5(repeat(tsvector::text, 10))::tsvector;')
node.psql('postgres', 'vacuum t_heap')
id = self.backup_pb(node, backup_type='ptrack', options=['-j100', '--stream'])
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['-j100', '--stream'])
node.psql('postgres', 'checkpoint')
for i in idx_ptrack:
@ -71,7 +75,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
node.psql('postgres', 'vacuum t_heap')
# Make page backup to clean every ptrack
self.backup_pb(node, backup_type='page', options=['-j100'])
self.backup_node(backup_dir, 'node', node, backup_type='page', options=['-j100'])
node.psql('postgres', 'checkpoint')
for i in idx_ptrack:
@ -85,8 +89,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack bits are cleaned
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size'])
print self.show_pb(node, as_text=True)
self.clean_pb(node)
print self.show_pb(backup_dir, 'node', as_text=True)
node.stop()
if __name__ == '__main__':

67
tests/ptrack_cluster.py
View File

@ -1,4 +1,5 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_cluster'
def teardown(self):
# clean_all()
@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_cluster_btree(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -46,8 +51,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'cluster t_heap using t_btree')
@ -68,17 +72,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
def test_ptrack_cluster_spgist(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -103,8 +109,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'cluster t_heap using t_spgist')
@ -125,17 +130,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
def test_ptrack_cluster_brin(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -160,8 +167,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'cluster t_heap using t_brin')
@ -182,17 +188,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
def test_ptrack_cluster_gist(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -217,8 +225,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'cluster t_heap using t_gist')
@ -239,17 +246,19 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
def test_ptrack_cluster_gin(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -274,8 +283,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'cluster t_heap using t_gin')
@ -296,7 +304,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

13
tests/ptrack_move_to_tablespace.py
View File

@ -10,6 +10,7 @@ from time import sleep
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_move_to_tablespace'
def teardown(self):
# clean_all()
@ -18,13 +19,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_ptrack_recovery(self):
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -54,7 +58,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack has correct bits after recovery
self.check_ptrack_recovery(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

13
tests/ptrack_recovery.py
View File

@ -10,6 +10,7 @@ from time import sleep
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_move_to_tablespace'
def teardown(self):
# clean_all()
@ -18,13 +19,16 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_ptrack_recovery(self):
fname = self.id().split(".")[3]
node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname),
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table
@ -56,7 +60,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# check that ptrack has correct bits after recovery
self.check_ptrack_recovery(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

15
tests/ptrack_vacuum.py
View File

@ -1,4 +1,5 @@
import unittest
import os
from sys import exit
from testgres import get_new_node, stop_all
from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum'
def teardown(self):
# clean_all()
@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_vacuum(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -47,8 +52,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
# Make full backup to clean every ptrack
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
for i in idx_ptrack:
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
node, idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
@ -74,7 +78,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

16
tests/ptrack_vacuum_bits_frozen.py
View File

@ -1,3 +1,4 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_bits_frozen'
def teardown(self):
# clean_all()
@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_vacuum_bits_frozen(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -45,8 +50,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'vacuum freeze t_heap')
node.psql('postgres', 'checkpoint')
@ -65,8 +69,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

16
tests/ptrack_vacuum_bits_visibility.py
View File

@ -1,3 +1,4 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_bits_visibility'
def teardown(self):
# clean_all()
@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_vacuum_bits_visibility(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -45,8 +50,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'vacuum t_heap')
node.psql('postgres', 'checkpoint')
@ -65,8 +69,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

16
tests/ptrack_vacuum_full.py
View File

@ -1,3 +1,4 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
@ -8,6 +9,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_full'
def teardown(self):
# clean_all()
@ -17,12 +19,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_vacuum_full(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -47,8 +52,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id%2 = 1')
node.psql('postgres', 'vacuum full t_heap')
@ -68,8 +72,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity, the most important part
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

16
tests/ptrack_vacuum_truncate.py
View File

@ -1,3 +1,4 @@
import os
import unittest
from sys import exit
from testgres import get_new_node, stop_all
@ -7,6 +8,7 @@ from helpers.ptrack_helpers import ProbackupTest, idx_ptrack
class SimpleTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(SimpleTest, self).__init__(*args, **kwargs)
self.module_name = 'ptrack_vacuum_truncate'
def teardown(self):
# clean_all()
@ -16,12 +18,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# @unittest.expectedFailure
def test_ptrack_vacuum_truncate(self):
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums', '-A trust'],
initdb_params=['--data-checksums'],
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# Create table and indexes
@ -46,8 +51,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
self.init_pb(node)
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream'])
node.psql('postgres', 'delete from t_heap where id > 128;')
node.psql('postgres', 'vacuum t_heap')
@ -67,8 +71,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# compare pages and check ptrack sanity
self.check_ptrack_sanity(idx_ptrack[i])
self.clean_pb(node)
node.stop()
if __name__ == '__main__':

79
tests/replica.py
View File

@ -15,8 +15,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ReplicaTest, self).__init__(*args, **kwargs)
self.module_name = 'replica'
self.instance_1 = 'master'
self.instance_2 = 'slave'
# @classmethod
# def tearDownClass(cls):
@ -25,9 +23,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_replica_stream_full_backup(self):
"""
make full stream backup from replica
"""
"""make full stream backup from replica"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
@ -35,37 +31,31 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
)
master.start()
shutil.rmtree(backup_dir, ignore_errors=True)
self.init_pb(backup_dir)
instance = ''
self.add_instance(backup_dir, self.instance_1, master)
self.set_archiving(backup_dir, self.instance_1, master)
master.restart()
self.add_instance(backup_dir, 'master', master)
slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname))
slave_port = slave.port
slave.cleanup()
# FULL BACKUP
self.backup_node(backup_dir, self.instance_1, master, backup_type='full', options=['--stream'])
self.backup_node(backup_dir, 'master', master, backup_type='full', options=['--stream'])
master.psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
before = master.execute("postgres", "SELECT * FROM t_heap")
#PAGE BACKUP
self.backup_node(backup_dir, self.instance_1, master, backup_type='page', options=['--stream'])
self.restore_node(backup_dir, self.instance_1, slave.data_dir)
#FULL BACKUP
self.backup_node(backup_dir, 'master', master, options=['--stream'])
self.restore_node(backup_dir, 'master', slave)
slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port))
slave.append_conf('postgresql.auto.conf', 'hot_standby = on')
slave.append_conf('recovery.conf', "standby_mode = 'on'")
slave.append_conf('recovery.conf',
"primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port))
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port))
slave.start({"-t": "600"})
# Replica Ready
@ -74,34 +64,35 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
self.assertEqual(before, after)
# Make backup from replica
self.add_instance(backup_dir, self.instance_2, slave)
time.sleep(2)
self.add_instance(backup_dir, 'slave', slave)
#time.sleep(2)
self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in
self.backup_node(backup_dir, self.instance_2, slave, backup_type='full', options=['--stream', '--log-level=verbose']))
self.validate_pb(backup_dir, self.instance_2)
self.assertEqual('OK', self.show_pb(backup_dir, self.instance_2)[0]['Status'])
self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose',
'--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]))
self.validate_pb(backup_dir, 'slave')
self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status'])
@unittest.skip("skip")
# @unittest.skip("skip")
def test_replica_archive_full_backup(self):
"""
make full archive backup from replica
"""
"""make full archive backup from replica"""
fname = self.id().split('.')[3]
master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname),
set_archiving=True,
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'master', master)
self.set_archiving(backup_dir, 'master', master)
# force more frequent wal switch
master.append_conf('postgresql.auto.conf', 'archive_timeout = 10')
master.start()
slave = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/slave".format(fname))
slave_port = slave.port
slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname))
slave.cleanup()
self.assertEqual(self.init_pb(master), six.b(""))
self.backup_pb(node=master, backup_type='full', options=['--stream'])
self.backup_node(backup_dir, 'master', master)
master.psql(
"postgres",
@ -109,28 +100,30 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
before = master.execute("postgres", "SELECT * FROM t_heap")
id = self.backup_pb(master, backup_type='page', options=['--stream'])
self.restore_pb(backup_dir=self.backup_dir(master), data_dir=slave.data_dir)
backup_id = self.backup_node(backup_dir, 'master', master, backup_type='page')
self.restore_node(backup_dir, 'master', slave)
# Settings for Replica
slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port))
slave.append_conf('postgresql.auto.conf', 'hot_standby = on')
# Set Archiving for replica
self.set_archiving_conf(slave, replica=True)
#self.set_archiving_conf( slave, replica=True)
self.set_archiving(backup_dir, 'slave', slave, replica=True)
# Set Replica
slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port))
slave.append_conf('postgresql.auto.conf', 'hot_standby = on')
slave.append_conf('recovery.conf', "standby_mode = 'on'")
slave.append_conf('recovery.conf',
"primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port))
"primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port))
slave.start({"-t": "600"})
# Replica Started
# master.execute("postgres", "checkpoint")
# Check replica
after = slave.execute("postgres", "SELECT * FROM t_heap")
self.assertEqual(before, after)
# Make backup from replica
self.assertEqual(self.init_pb(slave), six.b(""))
self.backup_pb(slave, backup_type='full', options=['--archive-timeout=30'])
self.validate_pb(slave)
self.add_instance(backup_dir, 'slave', slave)
self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300',
'--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])
self.validate_pb(backup_dir, 'slave')

479
tests/restore_test.py
View File

@ -1,52 +1,57 @@
import unittest
import os
from os import path
import six
from helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import stop_all
import subprocess
from datetime import datetime
import shutil
from sys import exit
class RestoreTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(RestoreTest, self).__init__(*args, **kwargs)
self.module_name = 'restore'
@classmethod
def tearDownClass(cls):
stop_all()
@unittest.skip("skip")
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_restore_full_to_latest(self):
"""recovery to latest from full backup"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
node.stop({"-m": "immediate"})
node.cleanup()
# 1 - Test recovery from latest
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# 2 - Test that recovery.conf was created
recovery_conf = path.join(node.data_dir, "recovery.conf")
self.assertEqual(path.isfile(recovery_conf), True)
recovery_conf = os.path.join(node.data_dir, "recovery.conf")
self.assertEqual(os.path.isfile(recovery_conf), True)
node.start({"-t": "600"})
@ -55,35 +60,38 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_full_page_to_latest(self):
"""recovery to latest from full + page backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
backup_log.write(self.backup_pb(node))
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="page")
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
node.stop({"-m": "immediate"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -92,29 +100,33 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_to_timeline(self):
"""recovery to target timeline"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
self.backup_pb(node, backup_type="full")
backup_id = self.backup_node(backup_dir, 'node', node)
target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")])
node.stop({"-m": "immediate"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -122,14 +134,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="full")
self.backup_node(backup_dir, 'node', node, backup_type="full")
node.stop({"-m": "immediate"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=["-j", "4", "--timeline=%i" % target_tli]))
# Correct Backup must be choosen for restore
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"]
self.assertEqual(int(recovery_target_timeline), target_tli)
@ -141,22 +154,24 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_to_time(self):
"""recovery to target timeline"""
"""recovery to target time"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
node.pgbench_init(scale=2)
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
backup_id = self.backup_pb(node, backup_type="full")
backup_id = self.backup_node(backup_dir, 'node', node)
target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@ -166,8 +181,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup {0} completed.".format(backup_id)) in self.restore_pb(node,
options=["-j", "4", '--time="{0}"'.format(target_time)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -176,23 +192,26 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_to_xid(self):
"""recovery to target xid"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
self.backup_pb(node, backup_type="full")
backup_id = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
@ -211,14 +230,14 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# Enforce segment to be archived to ensure that recovery goes up to the
# wanted point. There is no way to ensure that all segments needed have
# been archived up to the xmin point saved earlier without that.
node.execute("postgres", "SELECT pg_switch_xlog()")
#node.execute("postgres", "SELECT pg_switch_xlog()")
node.stop({"-m": "fast"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=["-j", "4", '--xid=%s' % target_xid]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -227,42 +246,38 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_full_ptrack_archive(self):
"""recovery to latest from full + ptrack backups"""
"""recovery to latest from archive full+ptrack backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
if not is_ptrack:
node.stop()
self.skipTest("ptrack not supported")
return
node.append_conf("postgresql.conf", "ptrack_enable = on")
node.restart()
self.backup_pb(node, backup_type="full")
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="ptrack")
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
node.stop({"-m": "immediate"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -271,48 +286,44 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
def test_restore_full_ptrack_ptrack(self):
"""recovery to latest from full + ptrack + ptrack backups"""
# @unittest.skip("skip")
def test_restore_ptrack(self):
"""recovery to latest from archive full+ptrack+ptrack backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
if not is_ptrack:
node.stop()
self.skipTest("ptrack not supported")
return
node.append_conf("postgresql.conf", "ptrack_enable = on")
node.restart()
self.backup_pb(node, backup_type="full")
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="ptrack")
self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="ptrack")
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack")
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
node.stop({"-m": "immediate"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -321,38 +332,39 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
# @unittest.skip("skip")
def test_restore_full_ptrack_stream(self):
"""recovery in stream mode to latest from full + ptrack backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
if not is_ptrack:
node.stop()
self.skipTest("ptrack not supported")
return
self.backup_pb(node, backup_type="full", options=["--stream"])
node.pgbench_init(scale=2)
self.backup_node(backup_dir, 'node', node, options=["--stream"])
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
backup_id = self.backup_pb(node, backup_type="ptrack", options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"])
before = node.execute("postgres", "SELECT * FROM pgbench_branches")
node.stop()
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup {0} completed.".format(backup_id)) in
self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -361,28 +373,24 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_full_ptrack_under_load(self):
"""recovery to latest from full + ptrack backups with loads when ptrack backup do"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
wal_segment_size = self.guc_wal_segment_size(node)
node.pgbench_init(scale=2)
is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
if not is_ptrack:
node.stop()
self.skipTest("ptrack not supported")
return
node.restart()
self.backup_pb(node, backup_type="full")
node.pgbench_init(scale=2)
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(
stdout=subprocess.PIPE,
@ -390,7 +398,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
options=["-c", "4", "-T", "8"]
)
self.backup_pb(node, backup_type="ptrack", options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"])
pgbench.wait()
pgbench.stdout.close()
@ -402,10 +410,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.wrong_wal_clean(node, wal_segment_size)
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -416,27 +423,23 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_full_under_load_ptrack(self):
"""recovery to latest from full + page backups with loads when full backup do"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
wal_segment_size = self.guc_wal_segment_size(node)
node.pgbench_init(scale=2)
is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
if not is_ptrack:
node.stop()
self.skipTest("ptrack not supported")
return
node.restart()
# wal_segment_size = self.guc_wal_segment_size(node)
node.pgbench_init(scale=2)
pgbench = node.pgbench(
stdout=subprocess.PIPE,
@ -444,12 +447,12 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
options=["-c", "4", "-T", "8"]
)
self.backup_pb(node, backup_type="full")
self.backup_node(backup_dir, 'node', node)
pgbench.wait()
pgbench.stdout.close()
self.backup_pb(node, backup_type="ptrack", options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"])
bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
@ -458,37 +461,39 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop({"-m": "immediate"})
node.cleanup()
self.wrong_wal_clean(node, wal_segment_size)
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node, options=["-j", "4"]))
#self.wrong_wal_clean(node, wal_segment_size)
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
self.assertEqual(bbalance, delta)
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_to_xid_inclusive(self):
"""recovery with target inclusive false"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
self.backup_pb(node, backup_type="full")
backup_id = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
@ -507,17 +512,15 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
# Enforce segment to be archived to ensure that recovery goes up to the
# wanted point. There is no way to ensure that all segments needed have
# been archived up to the xmin point saved earlier without that.
node.execute("postgres", "SELECT pg_switch_xlog()")
# node.execute("postgres", "SELECT pg_switch_xlog()")
node.stop({"-m": "fast"})
node.cleanup()
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=[
"-j", "4",
'--xid=%s' % target_xid,
"--inclusive=false"]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -527,20 +530,22 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_with_tablespace_mapping_1(self):
"""recovery using tablespace-mapping option"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}
pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
# Create tablespace
tblspc_path = path.join(node.base_dir, "tblspc")
tblspc_path = os.path.join(node.base_dir, "tblspc")
os.makedirs(tblspc_path)
with node.connect("postgres") as con:
con.connection.autocommit = True
@ -550,82 +555,88 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
con.execute("INSERT INTO test VALUES (1)")
con.commit()
self.backup_pb(node)
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
backup_id = self.backup_node(backup_dir, 'node', node)
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
# 1 - Try to restore to existing directory
node.stop()
try:
self.restore_pb(node)
assertEqual(1, 0, 'Error is expected because restore destination is not empty')
self.restore_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir))
self.assertEqual(e.message,
'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# 2 - Try to restore to existing tablespace directory
shutil.rmtree(node.data_dir)
node.cleanup()
try:
self.restore_pb(node)
assertEqual(1, 0, 'Error is expected because restore tablespace destination is not empty')
self.restore_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertEqual(
e.message,
'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path))
self.assertEqual(e.message,
'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# 3 - Restore using tablespace-mapping
tblspc_path_new = path.join(node.base_dir, "tblspc_new")
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]))
tblspc_path_new = os.path.join(node.base_dir, "tblspc_new")
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
id = node.execute("postgres", "SELECT id FROM test")
self.assertEqual(id[0][0], 1)
res = node.execute("postgres", "SELECT id FROM test")
self.assertEqual(res[0][0], 1)
# 4 - Restore using tablespace-mapping using page backup
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
with node.connect("postgres") as con:
con.execute("INSERT INTO test VALUES (2)")
con.commit()
self.backup_pb(node, backup_type="page")
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
show_pb = self.show_pb(node)
show_pb = self.show_pb(backup_dir, 'node')
self.assertEqual(show_pb[1]['Status'], six.b("OK"))
self.assertEqual(show_pb[2]['Status'], six.b("OK"))
node.stop()
node.cleanup()
tblspc_path_page = path.join(node.base_dir, "tblspc_page")
tblspc_path_page = os.path.join(node.base_dir, "tblspc_page")
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
id = node.execute("postgres", "SELECT id FROM test OFFSET 1")
self.assertEqual(id[0][0], 2)
res = node.execute("postgres", "SELECT id FROM test OFFSET 1")
self.assertEqual(res[0][0], 2)
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_restore_with_tablespace_mapping_2(self):
"""recovery using tablespace-mapping option and page backup"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
# Full backup
self.backup_pb(node)
self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK"))
self.backup_node(backup_dir, 'node', node)
self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK"))
# Create tablespace
tblspc_path = path.join(node.base_dir, "tblspc")
tblspc_path = os.path.join(node.base_dir, "tblspc")
os.makedirs(tblspc_path)
with node.connect("postgres") as con:
con.connection.autocommit = True
@ -635,9 +646,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
con.commit()
# First page backup
self.backup_pb(node, backup_type="page")
self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(node)[1]['Mode'], six.b("PAGE"))
self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE"))
# Create tablespace table
with node.connect("postgres") as con:
@ -649,18 +660,18 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
con.commit()
# Second page backup
self.backup_pb(node, backup_type="page")
self.assertEqual(self.show_pb(node)[2]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(node)[2]['Mode'], six.b("PAGE"))
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK"))
self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE"))
node.stop()
node.cleanup()
tblspc_path_new = path.join(node.base_dir, "tblspc_new")
tblspc_path_new = os.path.join(node.base_dir, "tblspc_new")
self.assertTrue(six.b("INFO: Restore of backup") and
six.b("completed.") in self.restore_pb(node,
options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start()
count = node.execute("postgres", "SELECT count(*) FROM tbl")
@ -669,30 +680,31 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.assertEqual(count[0][0], 4)
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_archive_node_backup_stream_restore_to_recovery_time(self):
"""make node with archiving, make stream backup, make PITR to Recovery Time
"""
"""make node with archiving, make stream backup, make PITR to Recovery Time"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/pgpro668/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
backup_id = self.backup_pb(node, backup_type='full', options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"])
node.psql("postgres", "create table t_heap(a int)")
node.psql("postgres", "select pg_switch_xlog()")
node.stop()
node.cleanup()
recovery_time = self.show_pb(node, id=backup_id)['recovery-time']
recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time']
self.assertTrue(six.b("INFO: Restore of backup {0} completed.".format(backup_id)) in
self.restore_pb(node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -701,29 +713,31 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.assertEqual(True, node.status())
node.stop()
@unittest.skip("skip")
# @unittest.skip("skip")
def test_archive_node_backup_stream_pitr(self):
"""make node with archiving, make stream backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
backup_id = self.backup_pb(node, backup_type='full', options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"])
node.psql("postgres", "create table t_heap(a int)")
node.psql("postgres", "select pg_switch_xlog()")
node.stop()
node.cleanup()
recovery_time = self.show_pb(node, id=backup_id)['recovery-time']
recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time']
self.assertTrue(six.b("INFO: Restore of backup {0} completed.".format(backup_id)) in
self.restore_pb(node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
@ -731,33 +745,34 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
self.assertEqual(True, 'does not exist' in res[2])
node.stop()
@unittest.skip("fucking illegal")
def test_archive_node_backup_stream_pitr_2(self):
"""make node with archiving, make stream backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists"""
# @unittest.skip("skip")
def test_archive_node_backup_archive_pitr_2(self):
"""make node with archiving, make archive backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname),
set_archiving=True,
set_replication=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
# exit(1)
self.assertEqual(self.init_pb(node), six.b(""))
backup_id = self.backup_pb(node, backup_type='full', options=["--stream"])
backup_id = self.backup_node(backup_dir, 'node', node)
node.psql("postgres", "create table t_heap(a int)")
node.psql("postgres", "select pg_switch_xlog()")
node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)})
node.cleanup()
recovery_time = self.show_pb(node, id=backup_id)['recovery-time']
recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
self.assertTrue(six.b("INFO: Restore of backup {0} completed.".format(backup_id)) in self.restore_pb(node,
options=["-j", "4", '--time="{0}"'.format(recovery_time)]))
self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)),
self.restore_node(backup_dir, 'node', node,
options=["-j", "4", '--time="{0}"'.format(recovery_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
node.start({"-t": "600"})
res = node.psql("postgres", 'select * from t_heap')
self.assertEqual(True, 'does not exist' in res[2])
# node.stop()
node.stop()

57
tests/retention_test.py
View File

@ -10,6 +10,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(RetentionTest, self).__init__(*args, **kwargs)
self.module_name = 'retention'
@classmethod
def tearDownClass(cls):
@ -20,30 +21,31 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
def test_retention_redundancy_1(self):
"""purge backups using redundancy-based retention policy"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.init_pb(node)
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf:
conf.write("retention-redundancy = 1\n")
# Make backups to be purged
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node)
self.backup_node(backup_dir, 'node', node, backup_type="page")
# Make backups to be keeped
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node)
self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(len(self.show_pb(node)), 4)
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4)
# Purge backups
log = self.delete_expired(node)
self.assertEqual(len(self.show_pb(node)), 2)
log = self.delete_expired(backup_dir, 'node')
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
# Check that WAL segments were deleted
min_wal = None
@ -53,7 +55,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
min_wal = line[31:-1]
elif line.startswith(b"INFO: removed max WAL segment"):
max_wal = line[31:-1]
for wal_name in listdir(path.join(self.backup_dir(node), "wal")):
for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')):
if not wal_name.endswith(".backup"):
wal_name_b = wal_name.encode('ascii')
self.assertEqual(wal_name_b[8:] > min_wal[8:], True)
@ -65,40 +67,43 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
def test_retention_window_2(self):
"""purge backups using window-based retention policy"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.init_pb(node)
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf:
conf.write("retention-redundancy = 1\n")
conf.write("retention-window = 1\n")
# Make backups to be purged
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node)
self.backup_node(backup_dir, 'node', node, backup_type="page")
# Make backup to be keeped
self.backup_pb(node)
self.backup_node(backup_dir, 'node', node)
backups = path.join(self.backup_dir(node), "backups")
backups = path.join(backup_dir, 'backups', 'node')
days_delta = 5
for backup in listdir(backups):
if backup == 'pg_probackup.conf':
continue
with open(path.join(backups, backup, "backup.control"), "a") as conf:
conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=days_delta)))
days_delta -= 1
# Make backup to be keeped
self.backup_pb(node, backup_type="page")
self.backup_node(backup_dir, 'node', node, backup_type="page")
self.assertEqual(len(self.show_pb(node)), 4)
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4)
# Purge backups
self.delete_expired(node)
self.assertEqual(len(self.show_pb(node)), 2)
self.delete_expired(backup_dir, 'node')
self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
node.stop()

38
tests/show_test.py
View File

@ -10,6 +10,7 @@ class OptionTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(OptionTest, self).__init__(*args, **kwargs)
self.module_name = 'show'
@classmethod
def tearDownClass(cls):
@ -17,39 +18,48 @@ class OptionTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def show_test_1(self):
def test_show_1(self):
"""Status DONE and OK"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname),
set_archiving=True,
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.assertEqual(
self.backup_pb(node, options=["--quiet"]),
self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]),
None
)
self.assertIn(six.b("OK"), self.show_pb(node, as_text=True))
self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True))
node.stop()
# @unittest.skip("skip")
def test_corrupt_2(self):
"""Status CORRUPT"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname),
set_archiving=True,
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
id_backup = self.backup_pb(node)
path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf")
os.remove(path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf"))
backup_id = self.backup_node(backup_dir, 'node', node)
self.validate_pb(node, id_backup)
self.assertIn(six.b("CORRUPT"), self.show_pb(node, as_text=True))
# delete file which belong to backup
file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf")
os.remove(file)
self.validate_pb(backup_dir, 'node', backup_id)
self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True))
node.stop()

148
tests/validate_test.py
View File

@ -13,6 +13,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ValidateTest, self).__init__(*args, **kwargs)
self.module_name = 'validate'
@classmethod
def tearDownClass(cls):
@ -23,20 +24,22 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
def test_validate_wal_unreal_values(self):
"""make node with archiving, make archive backup, validate to both real and unreal values"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
node.pgbench_init(scale=2)
pgbench = node.pgbench(
@ -48,17 +51,18 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
pgbench.wait()
pgbench.stdout.close()
target_time = self.show_pb(node, id=backup_id)['recovery-time']
target_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time']
after_backup_time = datetime.now().replace(second=0, microsecond=0)
# Validate to real time
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, options=["--time='{0}'".format(target_time)]))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# Validate to unreal time
unreal_time_1 = after_backup_time - timedelta(days=2)
try:
self.validate_pb(node, options=["--time='{0}'".format(unreal_time_1)])
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -68,7 +72,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Validate to unreal time #2
unreal_time_2 = after_backup_time + timedelta(days=2)
try:
self.validate_pb(node, options=["--time='{0}'".format(unreal_time_2)])
self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -83,13 +87,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
target_xid = res[0][0]
node.execute("postgres", "SELECT pg_switch_xlog()")
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, options=["--xid={0}".format(target_xid)]))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# Validate to unreal xid
unreal_xid = int(target_xid) + 1000
try:
self.validate_pb(node, options=["--xid={0}".format(unreal_xid)])
self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)])
self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
@ -97,35 +102,32 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
# Validate with backup ID
self.assertIn(six.b("INFO: backup validation completed successfully on"),
self.validate_pb(node, backup_id))
self.assertIn(six.b("INFO: backup validation completed successfully"),
self.validate_pb(backup_dir, 'node', backup_id),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
# @unittest.skip("skip")
def test_validate_corrupt_wal_1(self):
"""
make node with archiving
make archive backup
corrupt all wal files
run validate, expecting error because of wal corruption
make sure that backup status is 'CORRUPT'
"""
"""make archive node, make archive backup, corrupt all wal files, run validate, expect errors"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
# Corrupt WAL
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
@ -136,48 +138,43 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Simple validate
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# @unittest.skip("skip")
def test_validate_corrupt_wal_2(self):
"""
make node with archiving
make archive backup
corrupt all wal files
run validate to real xid, expecting error because of wal corruption
make sure that backup status is 'CORRUPT'
"""
"""make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
with node.connect("postgres") as con:
con.execute("CREATE TABLE tbl0005 (a text)")
con.commit()
backup_id = self.backup_pb(node)
backup_id = self.backup_node(backup_dir, 'node', node)
target_xid = None
with node.connect("postgres") as con:
res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
con.commit()
target_xid = res[0][0]
node.execute("postgres", "SELECT pg_switch_xlog()")
# Corrupt WAL
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals.sort()
for wal in wals:
@ -188,34 +185,34 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Validate to xid
try:
self.validate_pb(node, backup_id, options=['--xid={0}'.format(target_xid)])
self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)])
self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue('Possible WAL CORRUPTION' in e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"')
node.stop()
# @unittest.skip("skip")
def test_validate_wal_lost_segment_1(self):
"""
make node with archiving
make archive backup
"""make archive node, make archive full backup,
delete from archive wal segment which belong to previous backup
run validate, expecting error because of missing wal segment
make sure that backup status is 'CORRUPT'
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
node.pgbench_init(scale=2)
pgbench = node.pgbench(
stdout=subprocess.PIPE,
@ -224,31 +221,30 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
)
pgbench.wait()
pgbench.stdout.close()
backup_id = self.backup_pb(node, backup_type='full')
backup_id = self.backup_node(backup_dir, 'node', node)
# Delete wal segment
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
file = os.path.join(self.backup_dir(node), "wal", wals[1])
file = os.path.join(backup_dir, 'wal', 'node', wals[1])
os.remove(file)
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore from [0-9a-fA-F\/]+ to [0-9a-fA-F\/]+\n\Z'.format(
file), e.message),
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format(
file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual('CORRUPT', self.show_pb(node, id=backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"')
self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"')
# Be paranoid and run validate again
try:
self.validate_pb(node)
self.validate_pb(backup_dir, 'node')
self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('ERROR: Backup {0} has status: CORRUPT\n\Z'.format(backup_id), e.message),
self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
node.stop()
@ -262,15 +258,17 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
make sure that backup status is 'ERROR'
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname),
set_archiving=True,
node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.assertEqual(self.init_pb(node), six.b(""))
self.backup_pb(node, backup_type='full')
self.backup_node(backup_dir, 'node', node)
# make some wals
node.pgbench_init(scale=2)
@ -283,20 +281,22 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
pgbench.stdout.close()
# delete last wal segment
wals_dir = os.path.join(self.backup_dir(node), "wal")
wals_dir = os.path.join(backup_dir, 'wal', 'node')
wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')]
wals = map(int, wals)
file = os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))
file = os.path.join(wals_dir, '0000000' + str(max(wals)))
os.remove(file)
try:
backup_id = self.backup_pb(node, backup_type='page')
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format(
self.output, self.cmd))
except ProbackupException, e:
self.assertTrue(re.match('INFO: wait for LSN [0-9a-fA-F\/]+ in archived WAL segment .*\nWARNING: could not read WAL record at [0-9a-fA-F\/]+\nERROR: WAL segment "{0}" is absent\n\Z'.format(
file), e.message),
self.assertTrue('INFO: wait for LSN'
and 'in archived WAL segment'
and 'WARNING: could not read WAL record at'
and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
self.assertEqual('ERROR', self.show_pb(node)[1]['Status'], 'Backup {0} should have STATUS "ERROR"')
self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"')
node.stop()