import unittest
import os
from time import sleep
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from .helpers.cfs_helpers import find_by_name
import shutil
module_name = 'external'
class ExternalTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_basic_external(self):
"""
make node, create external directory, take backup
with external directory, restore backup, check that
external directory was successfully copied
"""
fname = self.id().split('.')[3]
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
set_replication=True)
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
external_dir = self.get_tblspace_path(node, 'somedirectory')
# create directory in external_directory
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# take FULL backup with external directory pointing to a file
file_path = os.path.join(core_dir, 'file')
open(file_path, "w+")
try:
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=[
'--external-dirs={0}'.format(file_path)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because external dir point to a file"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: --external-dirs option' in e.message and
'directory or symbolic link expected' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
sleep(1)
# FULL backup
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
# Fill external directories
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir, options=["-j", "4"])
# Full backup with external dir
self.backup_node(
backup_dir, 'node', node,
options=[
'--external-dirs={0}'.format(external_dir)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node, options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_external_none(self):
"""
make node, create external directory, take backup
with external directory, take delta backup with --external-dirs=none,
restore delta backup, check that
external directory was not copied
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
set_replication=True)
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
external_dir = self.get_tblspace_path(node, 'somedirectory')
# create directory in external_directory
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
# FULL backup
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
# Fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir, options=["-j", "4"])
# Full backup with external dir
self.backup_node(
backup_dir, 'node', node,
options=[
'--stream',
'--external-dirs={0}'.format(external_dir)])
# Delta backup without external directory
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=['--external-dirs=none', '--stream'])
shutil.rmtree(external_dir, ignore_errors=True)
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node, options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_external_dirs_overlapping(self):
"""
make node, create directory,
take backup with two external directories pointing to
the same directory, backup should fail
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'],
set_replication=True)
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# create directory in external_directory
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
os.mkdir(external_dir1)
os.mkdir(external_dir2)
# Full backup with external dirs
try:
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}{1}{0}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir1)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because tablespace mapping is incorrect"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: External directory path (-E option)' in e.message and
'contain another external directory' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_external_dir_mapping(self):
"""
make node, take full backup, check that restore with
external-dir mapping will end with error, take page backup,
check that restore with external-dir mapping will end with
success
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# Fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2')
try:
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(
external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(
external_dir2, external_dir2_new)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because tablespace mapping is incorrect"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'ERROR: --external-mapping option' in e.message and
'have an entry in list of external directories' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(
external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(
external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_backup_multiple_external(self):
"""check that cmdline has priority over config"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# FULL backup
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.set_config(
backup_dir, 'node',
options=['-E', external_dir1])
# cmdline option MUST override options in config
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", external_dir2])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs', 'external_dir1'])
node.cleanup()
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_backward_compatibility(self):
"""
take backup with old binary without external dirs support
take delta backup with new binary and 2 external directories
restore delta backup, check that incremental chain
restored correctly
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir, old_binary=True)
self.show_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
self.show_pb(backup_dir)
node.slow_start()
node.pgbench_init(scale=3)
# FULL backup with old binary without external dirs support
self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
# fill external directories with changed data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# delta backup with external directories using new binary
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# RESTORE chain with new binary
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_backward_compatibility_merge_1(self):
"""
take backup with old binary without external dirs support
take delta backup with new binary and 2 external directories
merge delta backup ajd restore it
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir, old_binary=True)
self.show_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
self.show_pb(backup_dir)
node.slow_start()
node.pgbench_init(scale=3)
# tmp FULL backup with old binary
tmp_id = self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# FULL backup with old binary without external dirs support
self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
pgbench = node.pgbench(options=['-T', '30', '-c', '1'])
pgbench.wait()
# delta backup with external directories using new binary
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge chain chain with new binary
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# Restore merged backup
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_backward_compatibility_merge_2(self):
"""
take backup with old binary without external dirs support
take delta backup with new binary and 2 external directories
merge delta backup and restore it
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir, old_binary=True)
self.show_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
self.show_pb(backup_dir)
node.slow_start()
node.pgbench_init(scale=3)
# tmp FULL backup with old binary
tmp_id = self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# FULL backup with old binary without external dirs support
self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
pgbench = node.pgbench(options=['-T', '30', '-c', '1'])
pgbench.wait()
# delta backup with external directories using new binary
self.backup_node(
backup_dir, 'node', node,
backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgbench = node.pgbench(options=['-T', '30', '-c', '1'])
pgbench.wait()
# Fill external dirs with changed data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1,
options=['-j', '4', '--skip-external-dirs'])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2,
options=['-j', '4', '--skip-external-dirs'])
# delta backup without external directories using old binary
backup_id = self.backup_node(
backup_dir, 'node', node,
backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge chain using new binary
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# Restore merged backup
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
external_dir1_new = self.get_tblspace_path(
node_restored, 'external_dir1')
external_dir2_new = self.get_tblspace_path(
node_restored, 'external_dir2')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(
external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(
external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_merge(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node, old_binary=True)
node.slow_start()
node.pgbench_init(scale=3)
# take temp FULL backup
tmp_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# fill external directories with data
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# FULL backup with old binary without external dirs support
self.backup_node(
backup_dir, 'node', node,
old_binary=True, options=["-j", "4", "--stream"])
# change data a bit
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# delta backup with external directories using new binary
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
external_dir1_new = self.get_tblspace_path(node, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node, 'external_dir2')
self.restore_node(
backup_dir, 'node', node,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(
external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(
external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_merge_skip_external_dirs(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=3)
# FULL backup with old data
tmp_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# fill external directories with old data
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# change data a bit
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup with external directories
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# drop old external data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
# fill external directories with new data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1,
options=["-j", "4", "--skip-external-dirs"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2,
options=["-j", "4", "--skip-external-dirs"])
# DELTA backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# merge backups without external directories
self.merge_backup(
backup_dir, 'node',
backup_id=backup_id, options=['--skip-external-dirs'])
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_merge_1(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=3)
# FULL backup
self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup with changed data
backup_id = self.backup_node(
backup_dir, 'node', node,
options=["-j", "4", "--stream"])
# fill external directories with changed data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# delta backup with external directories using new binary
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
external_dir1_new = self.get_tblspace_path(node, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node, 'external_dir2')
self.restore_node(
backup_dir, 'node', node,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_merge_3(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=3)
# FULL backup
self.backup_node(backup_dir, 'node', node, options=["-j", "4"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node)
# fill external directories with changed data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2)
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# page backup with external directories
self.backup_node(
backup_dir, 'node', node, backup_type="page",
options=[
"-j", "4",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# page backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="page",
options=[
"-j", "4",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.merge_backup(
backup_dir, 'node', backup_id=backup_id,
options=['--log-level-file=verbose'])
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
external_dir1_new = self.get_tblspace_path(node, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node, 'external_dir2')
self.restore_node(
backup_dir, 'node', node,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(
external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(
external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_merge_2(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=3)
# FULL backup
self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node,
options=["-j", "4", "--stream"])
# fill external directories with changed data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# delta backup with external directories
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# delta backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
# delta backup without external directories
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
external_dir1_new = self.get_tblspace_path(node, 'external_dir1')
external_dir2_new = self.get_tblspace_path(node, 'external_dir2')
self.restore_node(
backup_dir, 'node', node,
options=[
"-j", "4",
"--external-mapping={0}={1}".format(external_dir1, external_dir1_new),
"--external-mapping={0}={1}".format(external_dir2, external_dir2_new)])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_restore_external_changed_data(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=2)
# set externals
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# FULL backup
tmp_id = self.backup_node(
backup_dir, 'node',
node, options=["-j", "4", "--stream"])
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# change data a bit
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# fill external directories with changed data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
# change data a bit more
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# Delta backup with external directories
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Restore
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_restore_external_changed_data_1(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off',
'max_wal_size': '32MB'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=1)
# set externals
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# FULL backup
tmp_id = self.backup_node(
backup_dir, 'node',
node, options=["-j", "4", "--stream"])
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# change data a bit
pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# fill external directories with changed data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
# change data a bit more
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# Delta backup with only one external directory
self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", external_dir1])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs', 'external_dir2'])
# Restore
node.cleanup()
shutil.rmtree(node._base_dir)
# create empty file in external_dir2
os.mkdir(node._base_dir)
os.mkdir(external_dir2)
with open(os.path.join(external_dir2, 'file'), 'w+') as f:
f.close()
output = self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
self.assertNotIn(
'externaldir2',
output)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs', 'external_dir2'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_merge_external_changed_data(self):
""""""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off',
'max_wal_size': '32MB'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=2)
# set externals
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# FULL backup
tmp_id = self.backup_node(
backup_dir, 'node',
node, options=["-j", "4", "--stream"])
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# change data a bit
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# fill external directories with changed data
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, backup_id=backup_id,
options=["-j", "4", "--skip-external-dirs"])
# change data a bit more
pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum'])
pgbench.wait()
# Delta backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta",
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge
self.merge_backup(backup_dir, 'node', backup_id)
# Restore
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_restore_skip_external(self):
"""
Check that --skip-external-dirs works correctly
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir1 = self.get_tblspace_path(node, 'external_dir1')
external_dir2 = self.get_tblspace_path(node, 'external_dir2')
# temp FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
# fill external directories with data
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node,
data_dir=external_dir2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# FULL backup with external directories
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir2)])
# delete first externals, so pgdata_compare
# will be capable of detecting redundant
# external files after restore
shutil.rmtree(external_dir1, ignore_errors=True)
shutil.rmtree(external_dir2, ignore_errors=True)
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# RESTORE
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--skip-external-dirs"])
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_dir_is_symlink(self):
"""
Check that backup works correctly if external dir is symlink,
symlink pointing to external dir should be followed,
but restored as directory
"""
if os.name == 'nt':
return unittest.skip('Skipped for Windows')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# temp FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
# fill some directory with data
core_dir = os.path.join(self.tmp_path, module_name, fname)
symlinked_dir = os.path.join(core_dir, 'symlinked')
self.restore_node(
backup_dir, 'node', node,
data_dir=symlinked_dir, options=["-j", "4"])
# drop temp FULL backup
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# create symlink to directory in external directory
os.symlink(symlinked_dir, external_dir)
# FULL backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
# RESTORE
node_restored.cleanup()
external_dir_new = self.get_tblspace_path(
node_restored, 'external_dir')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4", "--external-mapping={0}={1}".format(
external_dir, external_dir_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
self.assertEqual(
external_dir,
self.show_pb(
backup_dir, 'node',
backup_id=backup_id)['external-dirs'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_dir_contain_symlink_on_dir(self):
"""
Check that backup works correctly if external dir is symlink,
symlink pointing to external dir should be followed,
but restored as directory
"""
if os.name == 'nt':
return unittest.skip('Skipped for Windows')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
dir_in_external_dir = os.path.join(external_dir, 'dir')
# temp FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
# fill some directory with data
core_dir = os.path.join(self.tmp_path, module_name, fname)
symlinked_dir = os.path.join(core_dir, 'symlinked')
self.restore_node(
backup_dir, 'node', node,
data_dir=symlinked_dir, options=["-j", "4"])
# drop temp FULL backup
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# create symlink to directory in external directory
os.mkdir(external_dir)
os.symlink(symlinked_dir, dir_in_external_dir)
# FULL backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
# RESTORE
node_restored.cleanup()
external_dir_new = self.get_tblspace_path(
node_restored, 'external_dir')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4", "--external-mapping={0}={1}".format(
external_dir, external_dir_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
self.assertEqual(
external_dir,
self.show_pb(
backup_dir, 'node',
backup_id=backup_id)['external-dirs'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_dir_contain_symlink_on_file(self):
"""
Check that backup works correctly if external dir is symlink,
symlink pointing to external dir should be followed,
but restored as directory
"""
if os.name == 'nt':
return unittest.skip('Skipped for Windows')
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
file_in_external_dir = os.path.join(external_dir, 'file')
# temp FULL backup
backup_id = self.backup_node(
backup_dir, 'node', node, options=["-j", "4", "--stream"])
# fill some directory with data
core_dir = os.path.join(self.tmp_path, module_name, fname)
symlinked_dir = os.path.join(core_dir, 'symlinked')
self.restore_node(
backup_dir, 'node', node,
data_dir=symlinked_dir, options=["-j", "4"])
# drop temp FULL backup
self.delete_pb(backup_dir, 'node', backup_id=backup_id)
# create symlink to directory in external directory
src_file = os.path.join(symlinked_dir, 'postgresql.conf')
os.mkdir(external_dir)
os.chmod(external_dir, 0700)
os.symlink(src_file, file_in_external_dir)
# FULL backup with external directories
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
# RESTORE
node_restored.cleanup()
external_dir_new = self.get_tblspace_path(
node_restored, 'external_dir')
self.restore_node(
backup_dir, 'node', node_restored,
options=[
"-j", "4", "--external-mapping={0}={1}".format(
external_dir, external_dir_new)])
pgdata_restored = self.pgdata_content(
node_restored.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
self.assertEqual(
external_dir,
self.show_pb(
backup_dir, 'node',
backup_id=backup_id)['external-dirs'])
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.expectedFailure
# @unittest.skip("skip")
def test_external_dir_is_tablespace(self):
"""
Check that backup fails with error
if external directory points to tablespace
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
self.create_tblspace_in_node(
node, 'tblspace1', tblspc_path=external_dir)
node.pgbench_init(scale=1, tablespace='tblspace1')
# FULL backup with external directories
try:
backup_id = self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because external dir points to the tablespace"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'External directory path (-E option)',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_restore_external_dir_not_empty(self):
"""
Check that backup fails with error
if external directory point to not empty tablespace and
if remapped directory also isn`t empty
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# create empty file in external directory
# open(os.path.join(external_dir, 'file'), 'a').close()
os.mkdir(external_dir)
with open(os.path.join(external_dir, 'file'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
node.cleanup()
try:
self.restore_node(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because external dir is not empty"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'External directory is not empty',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
external_dir_new = self.get_tblspace_path(node, 'external_dir_new')
# create empty file in directory, which will be a target of
# remapping
os.mkdir(external_dir_new)
with open(os.path.join(external_dir_new, 'file1'), 'w+') as f:
f.close()
try:
self.restore_node(
backup_dir, 'node', node,
options=['--external-mapping={0}={1}'.format(
external_dir, external_dir_new)])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because remapped external dir is not empty"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'External directory is not empty',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_restore_external_dir_is_missing(self):
"""
take FULL backup with not empty external directory
delete external directory
take DELTA backup with external directory, which
should fail
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# create empty file in external directory
# open(os.path.join(external_dir, 'file'), 'a').close()
os.mkdir(external_dir)
with open(os.path.join(external_dir, 'file'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
# drop external directory
shutil.rmtree(external_dir, ignore_errors=True)
try:
self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", external_dir])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because external dir is missing"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'ERROR: External directory is not found:',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
sleep(1)
# take DELTA without external directories
self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=["-j", "4", "--stream"])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Restore Delta backup
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_merge_external_dir_is_missing(self):
"""
take FULL backup with not empty external directory
delete external directory
take DELTA backup with external directory, which
should fail,
take DELTA backup without external directory,
merge it into FULL, restore and check
data correctness
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# create empty file in external directory
# open(os.path.join(external_dir, 'file'), 'a').close()
os.mkdir(external_dir)
with open(os.path.join(external_dir, 'file'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
# drop external directory
shutil.rmtree(external_dir, ignore_errors=True)
try:
self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", external_dir])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because external dir is missing"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'ERROR: External directory is not found:',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
sleep(1)
# take DELTA without external directories
backup_id = self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=["-j", "4", "--stream"])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# Restore
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_restore_external_dir_is_empty(self):
"""
take FULL backup with not empty external directory
drop external directory content
take DELTA backup with the same external directory
restore DELRA backup, check that restored
external directory is empty
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# create empty file in external directory
# open(os.path.join(external_dir, 'file'), 'a').close()
os.mkdir(external_dir)
os.chmod(external_dir, 0700)
with open(os.path.join(external_dir, 'file'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
# make external directory empty
os.remove(os.path.join(external_dir, 'file'))
# take DELTA backup with empty external directory
self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", external_dir])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Restore Delta backup
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_merge_external_dir_is_empty(self):
"""
take FULL backup with not empty external directory
drop external directory content
take DELTA backup with the same external directory
merge backups and restore FULL, check that restored
external directory is empty
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir = self.get_tblspace_path(node, 'external_dir')
# create empty file in external directory
# open(os.path.join(external_dir, 'file'), 'a').close()
os.mkdir(external_dir)
os.chmod(external_dir, 0700)
with open(os.path.join(external_dir, 'file'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", external_dir])
# make external directory empty
os.remove(os.path.join(external_dir, 'file'))
# take DELTA backup with empty external directory
backup_id = self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", external_dir])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# Restore
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_restore_external_dir_string_order(self):
"""
take FULL backup with not empty external directory
drop external directory content
take DELTA backup with the same external directory
restore DELRA backup, check that restored
external directory is empty
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir_1 = self.get_tblspace_path(node, 'external_dir_1')
external_dir_2 = self.get_tblspace_path(node, 'external_dir_2')
# create empty file in external directory
os.mkdir(external_dir_1)
os.chmod(external_dir_1, 0700)
with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f:
f.close()
os.mkdir(external_dir_2)
os.chmod(external_dir_2, 0700)
with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir_1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir_2)])
with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f:
f.close()
with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f:
f.close()
# take DELTA backup and swap external_dir_2 and external_dir_1
# in external_dir_str
self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir_2,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir_1)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Restore Delta backup
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_merge_external_dir_string_order(self):
"""
take FULL backup with not empty external directory
drop external directory content
take DELTA backup with the same external directory
restore DELRA backup, check that restored
external directory is empty
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
core_dir = os.path.join(self.tmp_path, module_name, fname)
shutil.rmtree(core_dir, ignore_errors=True)
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
external_dir_1 = self.get_tblspace_path(node, 'external_dir_1')
external_dir_2 = self.get_tblspace_path(node, 'external_dir_2')
# create empty file in external directory
os.mkdir(external_dir_1)
os.chmod(external_dir_1, 0700)
with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f:
f.close()
os.mkdir(external_dir_2)
os.chmod(external_dir_2, 0700)
with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f:
f.close()
# FULL backup with external directory
self.backup_node(
backup_dir, 'node', node,
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir_1,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir_2)])
with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f:
f.close()
with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f:
f.close()
# take DELTA backup and swap external_dir_2 and external_dir_1
# in external_dir_str
backup_id = self.backup_node(
backup_dir, 'node', node,
backup_type='delta',
options=[
"-j", "4", "--stream",
"-E", "{0}{1}{2}".format(
external_dir_2,
self.EXTERNAL_DIRECTORY_DELIMITER,
external_dir_1)])
pgdata = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
# Merge backups
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
# Restore
node.cleanup()
shutil.rmtree(node.base_dir, ignore_errors=True)
self.restore_node(backup_dir, 'node', node)
pgdata_restored = self.pgdata_content(
node.base_dir, exclude_dirs=['logs'])
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_smart_restore_externals(self):
"""
make node, create database, take full backup with externals,
take incremental backup without externals and restore it,
make sure that files from externals are not copied during restore
https://github.com/postgrespro/pg_probackup/issues/63
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# fill external directories with data
tmp_id = self.backup_node(backup_dir, 'node', node)
external_dir_1 = self.get_tblspace_path(node, 'external_dir_1')
external_dir_2 = self.get_tblspace_path(node, 'external_dir_2')
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir_1, options=["-j", "4"])
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir_2, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# create database
node.safe_psql(
"postgres",
"CREATE DATABASE testdb")
# take FULL backup
full_id = self.backup_node(backup_dir, 'node', node)
# drop database
node.safe_psql(
"postgres",
"DROP DATABASE testdb")
# take PAGE backup
page_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
# restore PAGE backup
node.cleanup()
self.restore_node(
backup_dir, 'node', node, backup_id=page_id,
options=['--no-validate', '--log-level-file=VERBOSE'])
logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log')
with open(logfile, 'r') as f:
logfile_content = f.read()
# get delta between FULL and PAGE filelists
filelist_full = self.get_backup_filelist(
backup_dir, 'node', full_id)
filelist_page = self.get_backup_filelist(
backup_dir, 'node', page_id)
filelist_diff = self.get_backup_filelist_diff(
filelist_full, filelist_page)
for file in filelist_diff:
self.assertNotIn(file, logfile_content)
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_external_validation(self):
"""
make node, create database,
take full backup with external directory,
corrupt external file in backup,
run validate which should fail
"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
node.slow_start()
# take temp FULL backup
tmp_id = self.backup_node(
backup_dir, 'node', node, options=['--stream'])
external_dir = self.get_tblspace_path(node, 'external_dir')
# fill external directories with data
self.restore_node(
backup_dir, 'node', node, backup_id=tmp_id,
data_dir=external_dir, options=["-j", "4"])
self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
# take FULL backup
full_id = self.backup_node(
backup_dir, 'node', node,
options=[
'--stream', '-E', "{0}".format(external_dir)])
# Corrupt file
file = os.path.join(
backup_dir, 'backups', 'node', full_id,
'external_directories', 'externaldir1', 'postgresql.auto.conf')
with open(file, "r+b", 0) as f:
f.seek(42)
f.write(b"blah")
f.flush()
f.close
try:
self.validate_pb(backup_dir)
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because file in external dir is corrupted"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
'WARNING: Invalid CRC of backup file',
e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd))
self.assertEqual(
'CORRUPT',
self.show_pb(backup_dir, 'node', full_id)['status'],
'Backup STATUS should be "CORRUPT"')
# Clean after yourself
self.del_test_dir(module_name, fname)