1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-12-13 11:53:59 +02:00
pg_probackup/tests/cfs_backup.py
2018-12-25 17:48:49 +03:00

1162 lines
40 KiB
Python

import os
import unittest
import random
import shutil
from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
module_name = 'cfs_backup'
tblspace_name = 'cfs_tblspace'
class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
# --- Begin --- #
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def setUp(self):
self.fname = self.id().split('.')[3]
self.backup_dir = os.path.join(
self.tmp_path, module_name, self.fname, 'backup')
self.node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, self.fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'wal_level': 'replica',
'ptrack_enable': 'on',
'cfs_encryption': 'off',
'max_wal_senders': '2',
'shared_buffers': '200MB'
}
)
self.init_pb(self.backup_dir)
self.add_instance(self.backup_dir, 'node', self.node)
self.set_archiving(self.backup_dir, 'node', self.node)
self.node.slow_start()
self.create_tblspace_in_node(self.node, tblspace_name, cfs=True)
tblspace = self.node.safe_psql(
"postgres",
"SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(
tblspace_name)
)
self.assertTrue(
tblspace_name in tblspace and "compression=true" in tblspace,
"ERROR: The tablespace not created "
"or it create without compressions"
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
# --- Section: Full --- #
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace(self):
"""Case: Check fullbackup empty compressed tablespace"""
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Full backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['pg_compression']),
"ERROR: File pg_compression not found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace_stream(self):
"""Case: Check fullbackup empty compressed tablespace with options stream"""
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Full backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['pg_compression']),
"ERROR: File pg_compression not found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
# PGPRO-1018 invalid file size
def test_fullbackup_after_create_table(self):
"""Case: Make full backup after created table in the tablespace"""
if not self.enterprise:
return
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"\n ERROR: {0}\n CMD: {1}".format(
repr(e.message),
repr(self.cmd)
)
)
return False
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Full backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['pg_compression']),
"ERROR: File pg_compression not found in {0}".format(
os.path.join(self.backup_dir, 'node', backup_id))
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
# PGPRO-1018 invalid file size
def test_fullbackup_after_create_table_stream(self):
"""
Case: Make full backup after created table in the tablespace with option --stream
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Full backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['pg_compression']),
"ERROR: File pg_compression not found in backup dir"
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
# --- Section: Incremental from empty tablespace --- #
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace_ptrack_after_create_table(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table
"""
try:
self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='ptrack')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Incremental backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table
"""
try:
self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='ptrack', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Incremental backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
self.assertFalse(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['_ptrack']),
"ERROR: _ptrack files was found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace_page_after_create_table(self):
"""
Case: Make full backup before created table in the tablespace.
Make page backup after create table
"""
try:
self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='page')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Incremental backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_empty_tablespace_page_after_create_table_stream(self):
"""
Case: Make full backup before created table in the tablespace.
Make page backup after create table
"""
try:
self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
backup_id = None
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='page', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Incremental backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
self.assertFalse(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['_ptrack']),
"ERROR: _ptrack files was found in backup dir"
)
# --- Section: Incremental from fill tablespace --- #
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_after_create_table_ptrack_after_create_table(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table.
Check: incremental backup will not greater as full
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format('t1', tblspace_name)
)
backup_id_full = None
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format('t2', tblspace_name)
)
backup_id_ptrack = None
try:
backup_id_ptrack = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='ptrack')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_ptrack = self.show_pb(
self.backup_dir, 'node', backup_id_ptrack)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_ptrack["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_ptrack["data-bytes"],
show_backup_full["data-bytes"]
)
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self):
"""
Case: Make full backup before created table in the tablespace(--stream).
Make ptrack backup after create table(--stream).
Check: incremental backup size should not be greater than full
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format('t1', tblspace_name)
)
backup_id_full = None
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,25) i".format('t2', tblspace_name)
)
backup_id_ptrack = None
try:
backup_id_ptrack = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='ptrack', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_ptrack = self.show_pb(
self.backup_dir, 'node', backup_id_ptrack)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_ptrack["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_ptrack["data-bytes"],
show_backup_full["data-bytes"]
)
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_after_create_table_page_after_create_table(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table.
Check: incremental backup size should not be greater than full
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format('t1', tblspace_name)
)
backup_id_full = None
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format('t2', tblspace_name)
)
backup_id_page = None
try:
backup_id_page = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='page')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_page = self.show_pb(
self.backup_dir, 'node', backup_id_page)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_page["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_page["data-bytes"],
show_backup_full["data-bytes"]
)
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_multiple_segments(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table.
Check: incremental backup will not greater as full
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format(
't_heap', tblspace_name)
)
full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap")
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"INSERT INTO {0} "
"SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format(
't_heap')
)
page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap")
try:
backup_id_page = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='page')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_page = self.show_pb(
self.backup_dir, 'node', backup_id_page)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_page["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_page["data-bytes"],
show_backup_full["data-bytes"]
)
)
# CHECK FULL BACKUP
self.node.stop()
self.node.cleanup()
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name),
ignore_errors=True)
self.restore_node(
self.backup_dir, 'node', self.node,
backup_id=backup_id_full, options=["-j", "4"])
self.node.slow_start()
self.assertEqual(
full_result,
self.node.safe_psql("postgres", "SELECT * FROM t_heap"),
'Lost data after restore')
# CHECK PAGE BACKUP
self.node.stop()
self.node.cleanup()
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name),
ignore_errors=True)
self.restore_node(
self.backup_dir, 'node', self.node,
backup_id=backup_id_page, options=["-j", "4"])
self.node.slow_start()
self.assertEqual(
page_result,
self.node.safe_psql("postgres", "SELECT * FROM t_heap"),
'Lost data after restore')
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_multiple_segments_in_multiple_tablespaces(self):
"""
Case: Make full backup before created table in the tablespace.
Make ptrack backup after create table.
Check: incremental backup will not greater as full
"""
tblspace_name_1 = 'tblspace_name_1'
tblspace_name_2 = 'tblspace_name_2'
self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True)
self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format(
't_heap_1', tblspace_name_1)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format(
't_heap_2', tblspace_name_2)
)
full_result_1 = self.node.safe_psql(
"postgres", "SELECT * FROM t_heap_1")
full_result_2 = self.node.safe_psql(
"postgres", "SELECT * FROM t_heap_2")
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"INSERT INTO {0} "
"SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format(
't_heap_1')
)
self.node.safe_psql(
"postgres",
"INSERT INTO {0} "
"SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format(
't_heap_2')
)
page_result_1 = self.node.safe_psql(
"postgres", "SELECT * FROM t_heap_1")
page_result_2 = self.node.safe_psql(
"postgres", "SELECT * FROM t_heap_2")
try:
backup_id_page = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='page')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_page = self.show_pb(
self.backup_dir, 'node', backup_id_page)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_page["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_page["data-bytes"],
show_backup_full["data-bytes"]
)
)
# CHECK FULL BACKUP
self.node.stop()
self.node.cleanup()
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name_1),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name_2),
ignore_errors=True)
self.restore_node(
self.backup_dir, 'node', self.node,
backup_id=backup_id_full, options=["-j", "4"])
self.node.slow_start()
self.assertEqual(
full_result_1,
self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"),
'Lost data after restore')
self.assertEqual(
full_result_2,
self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"),
'Lost data after restore')
# CHECK PAGE BACKUP
self.node.stop()
self.node.cleanup()
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name_1),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(self.node, tblspace_name_2),
ignore_errors=True)
self.restore_node(
self.backup_dir, 'node', self.node,
backup_id=backup_id_page, options=["-j", "4"])
self.node.slow_start()
self.assertEqual(
page_result_1,
self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"),
'Lost data after restore')
self.assertEqual(
page_result_2,
self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"),
'Lost data after restore')
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_fullbackup_after_create_table_page_after_create_table_stream(self):
"""
Case: Make full backup before created table in the tablespace(--stream).
Make ptrack backup after create table(--stream).
Check: incremental backup will not greater as full
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,1005000) i".format('t1', tblspace_name)
)
backup_id_full = None
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='full', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,10) i".format('t2', tblspace_name)
)
backup_id_page = None
try:
backup_id_page = self.backup_node(
self.backup_dir, 'node', self.node,
backup_type='page', options=['--stream'])
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup_full = self.show_pb(
self.backup_dir, 'node', backup_id_full)
show_backup_page = self.show_pb(
self.backup_dir, 'node', backup_id_page)
self.assertGreater(
show_backup_full["data-bytes"],
show_backup_page["data-bytes"],
"ERROR: Size of incremental backup greater than full. \n "
"INFO: {0} >{1}".format(
show_backup_page["data-bytes"],
show_backup_full["data-bytes"]
)
)
# --- Make backup with not valid data(broken .cfm) --- #
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_delete_random_cfm_file_from_tablespace_dir(self):
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
list_cmf = find_by_extensions(
[self.get_tblspace_path(self.node, tblspace_name)],
['.cfm'])
self.assertTrue(
list_cmf,
"ERROR: .cfm-files not found into tablespace dir"
)
os.remove(random.choice(list_cmf))
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_delete_file_pg_compression_from_tablespace_dir(self):
os.remove(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression'])[0])
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_delete_random_data_file_from_tablespace_dir(self):
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
list_data_files = find_by_pattern(
[self.get_tblspace_path(self.node, tblspace_name)],
'^.*/\d+$')
self.assertTrue(
list_data_files,
"ERROR: Files of data not found into tablespace dir"
)
os.remove(random.choice(list_data_files))
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_broken_random_cfm_file_into_tablespace_dir(self):
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
list_cmf = find_by_extensions(
[self.get_tblspace_path(self.node, tblspace_name)],
['.cfm'])
self.assertTrue(
list_cmf,
"ERROR: .cfm-files not found into tablespace dir"
)
corrupt_file(random.choice(list_cmf))
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_broken_random_data_file_into_tablespace_dir(self):
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
list_data_files = find_by_pattern(
[self.get_tblspace_path(self.node, tblspace_name)],
'^.*/\d+$')
self.assertTrue(
list_data_files,
"ERROR: Files of data not found into tablespace dir"
)
corrupt_file(random.choice(list_data_files))
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
@unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_broken_file_pg_compression_into_tablespace_dir(self):
corrupted_file = find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression'])[0]
self.assertTrue(
corrupt_file(corrupted_file),
"ERROR: File is not corrupted or it missing"
)
self.assertRaises(
ProbackupException,
self.backup_node,
self.backup_dir,
'node',
self.node,
backup_type='full'
)
# # --- End ---#
# @unittest.skipUnless(ProbackupTest.enterprise, 'skip')
# def tearDown(self):
# self.node.cleanup()
# self.del_test_dir(module_name, self.fname)
#class CfsBackupEncTest(CfsBackupNoEncTest):
# # --- Begin --- #
# def setUp(self):
# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key"
# super(CfsBackupEncTest, self).setUp()