Add many tests and some words for docs.

README.md

@@ -55,10 +55,28 @@ All the documentation you can find [here](doc/pg_probackup.md).
 Regression tests
 ----------------
 
-The test suite of pg_probackup is available in the code tree and can be
-launched in a way similar to common PostgreSQL extensions and modules:
+For tests you must have python 2.7 or python 3.3 and higher. Also good idea
+is make virtual enviroment by `virtualenv`.   
+First of all you need to install `testgres` python module which contains useful
+functions to start postgres clusters and make queries:
+
+```
+pip install testgres
+```
+
+To run tests execute:
+
+```
+python -m unittest tests
+```
+
+from current (root of project) directory. If you want to run a specific postgres build then
+you should specify the path to your pg_config executable by setting PG_CONFIG
+environment variable:
+```
+export PG_CONFIG=/path/to/pg_config
+```
 
-    make installcheck
 
 Block level incremental backup
 ------------------------------

tests/__init__.py

@@ -1,6 +1,6 @@
 import unittest
 
-from . import init_test, option_test, show_test, backup_test, delete_test, restore_test
+from . import init_test, option_test, show_test, backup_test, delete_test, restore_test, validate_test
 
 
 def load_tests(loader, tests, pattern):
@@ -11,5 +11,6 @@ def load_tests(loader, tests, pattern):
 	suite.addTests(loader.loadTestsFromModule(backup_test))
 	suite.addTests(loader.loadTestsFromModule(delete_test))
 	suite.addTests(loader.loadTestsFromModule(restore_test))
+	suite.addTests(loader.loadTestsFromModule(validate_test))
 
 	return suite

tests/restore_test.py

@@ -246,7 +246,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
 
 	def test_restore_full_ptrack_6(self):
 		"""recovery to latest from full + ptrack backups"""
-		node = self.make_bnode('restore_full_ptrack_6', base_dir="tmp_dirs/restore/restore_full_ptrack_6")
+		node = self.make_bnode('restore_full_ptrack_6', base_dir="tmp_dirs/restore/full_ptrack_6")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -290,3 +290,283 @@ class RestoreTest(ProbackupTest, unittest.TestCase):
 		self.assertEqual(before, after)
 
 		node.stop()
+
+	def test_restore_full_ptrack_ptrack_7(self):
+		"""recovery to latest from full + ptrack + ptrack backups"""
+		node = self.make_bnode('restore_full_ptrack_ptrack_7', base_dir="tmp_dirs/restore/full_ptrack_ptrack_7")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
+		if not is_ptrack:
+			node.stop()
+			self.skipTest("ptrack not supported")
+			return
+
+		node.append_conf("postgresql.conf", "ptrack_enable = on")
+		node.restart()
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"]))
+
+		before = node.execute("postgres", "SELECT * FROM pgbench_branches")
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "immediate"
+		})
+
+		with open(path.join(node.logs_dir, "restore_1.log"), "wb") as restore_log:
+			restore_log.write(self.restore_pb(node, options=["-j", "4", "--verbose"]))
+
+		node.pg_ctl("start", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-t": "600"
+		})
+
+		after = node.execute("postgres", "SELECT * FROM pgbench_branches")
+		self.assertEqual(before, after)
+
+		node.stop()
+
+	def test_restore_full_ptrack_stream_8(self):
+		"""recovery in stream mode to latest from full + ptrack backups"""
+		node = self.make_bnode('restore_full_ptrack_stream_8', base_dir="tmp_dirs/restore/full_ptrack_stream_8")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
+		if not is_ptrack:
+			node.stop()
+			self.skipTest("ptrack not supported")
+			return
+
+		node.append_conf("postgresql.conf", "ptrack_enable = on")
+		node.restart()
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "--stream"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"]))
+
+		before = node.execute("postgres", "SELECT * FROM pgbench_branches")
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "immediate"
+		})
+
+		with open(path.join(node.logs_dir, "restore_1.log"), "wb") as restore_log:
+			restore_log.write(self.restore_pb(node, options=["-j", "4", "--verbose"]))
+
+		node.pg_ctl("start", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-t": "600"
+		})
+
+		after = node.execute("postgres", "SELECT * FROM pgbench_branches")
+		self.assertEqual(before, after)
+
+		node.stop()
+
+	def test_restore_full_ptrack_under_load_9(self):
+		"""recovery to latest from full + page backups with loads when ptrack backup do"""
+		node = self.make_bnode('restore_full_ptrack_under_load_9', base_dir="tmp_dirs/restore/full_ptrack_under_load_9")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
+		if not is_ptrack:
+			node.stop()
+			self.skipTest("ptrack not supported")
+			return
+
+		node.append_conf("postgresql.conf", "ptrack_enable = on")
+		node.restart()
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"]))
+
+		pgbench = node.pgbench(
+			stdout=subprocess.PIPE,
+			stderr=subprocess.STDOUT,
+			options=["-c", "4", "-T", "8"]
+		)
+
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"]))
+
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		node.execute("postgres", "SELECT pg_switch_xlog()")
+
+		bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
+		delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
+
+		self.assertEqual(bbalance, delta)
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "immediate"
+		})
+
+		with open(path.join(node.logs_dir, "restore_1.log"), "wb") as restore_log:
+			restore_log.write(self.restore_pb(node, options=["-j", "4", "--verbose"]))
+
+		node.pg_ctl("start", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-t": "600"
+		})
+
+		bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
+		delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
+
+		self.assertEqual(bbalance, delta)
+
+		node.stop()
+
+	def test_restore_full_under_load_ptrack_10(self):
+		"""recovery to latest from full + page backups with loads when full backup do"""
+		node = self.make_bnode('estore_full_under_load_ptrack_10', base_dir="tmp_dirs/restore/full_under_load_ptrack_10")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
+		if not is_ptrack:
+			node.stop()
+			self.skipTest("ptrack not supported")
+			return
+
+		node.append_conf("postgresql.conf", "ptrack_enable = on")
+		node.restart()
+
+		pgbench = node.pgbench(
+			stdout=subprocess.PIPE,
+			stderr=subprocess.STDOUT,
+			options=["-c", "4", "-T", "8"]
+		)
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"]))
+
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"]))
+
+		node.execute("postgres", "SELECT pg_switch_xlog()")
+
+		bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
+		delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
+
+		self.assertEqual(bbalance, delta)
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "immediate"
+		})
+
+		with open(path.join(node.logs_dir, "restore_1.log"), "wb") as restore_log:
+			restore_log.write(self.restore_pb(node, options=["-j", "4", "--verbose"]))
+
+		node.pg_ctl("start", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-t": "600"
+		})
+
+		bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches")
+		delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history")
+
+		self.assertEqual(bbalance, delta)
+
+		node.stop()
+
+	def test_restore_to_xid_inclusive_11(self):
+		"""recovery with target inclusive false"""
+		node = self.make_bnode('estore_to_xid_inclusive_11', base_dir="tmp_dirs/restore/restore_to_xid_inclusive_11")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		with node.connect("postgres") as con:
+			con.execute("CREATE TABLE tbl0005 (a text)")
+			con.commit()
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		before = node.execute("postgres", "SELECT * FROM pgbench_branches")
+		with node.connect("postgres") as con:
+			res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
+			con.commit()
+			target_xid = res[0][0]
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		# Enforce segment to be archived to ensure that recovery goes up to the
+		# wanted point. There is no way to ensure that all segments needed have
+		# been archived up to the xmin point saved earlier without that.
+		node.execute("postgres", "SELECT pg_switch_xlog()")
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "fast"
+		})
+
+		with open(path.join(node.logs_dir, "restore_1.log"), "wb") as restore_log:
+			restore_log.write(self.restore_pb(
+				node,
+				options=[
+					"-j", "4",
+					"--verbose",
+					'--xid=%s' % target_xid,
+					"--inclusive=false"
+				]
+			))
+
+		node.pg_ctl("start", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-t": "600"
+		})
+
+		after = node.execute("postgres", "SELECT * FROM pgbench_branches")
+		self.assertEqual(before, after)
+		self.assertEqual(len(node.execute("postgres", "SELECT * FROM tbl0005")), 0)
+
+		node.stop()

tests/validate_test.py

@@ -0,0 +1,66 @@
+import unittest
+from os import path, listdir
+import six
+from .pb_lib import ProbackupTest
+from testgres import stop_all
+import subprocess
+
+
+class ValidateTest(ProbackupTest, unittest.TestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(ValidateTest, self).__init__(*args, **kwargs)
+
+	@classmethod
+	def tearDownClass(cls):
+		try:
+			stop_all()
+		except:
+			pass
+
+	def test_validate_broke_wal_1(self):
+		"""recovery to latest from full backup"""
+		node = self.make_bnode('test_validate_broke_wal_1', base_dir="tmp_dirs/validate/broke_wal_1")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init(scale=2)
+		with node.connect("postgres") as con:
+			con.execute("CREATE TABLE tbl0005 (a text)")
+			con.commit()
+
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		pgbench = node.pgbench(
+			stdout=subprocess.PIPE,
+			stderr=subprocess.STDOUT,
+			options=["-c", "4", "-T", "10"]
+		)
+
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		target_xid = None
+		with node.connect("postgres") as con:
+			res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
+			con.commit()
+			target_xid = res[0][0]
+
+		node.execute("postgres", "SELECT pg_switch_xlog()")
+
+		node.pg_ctl("stop", {
+			"-D": node.data_dir,
+			"-w": None,
+			"-m": "immediate"
+		})
+
+		wals_dir = path.join(self.backup_dir(node), "wal")
+		wals = [f for f in listdir(wals_dir) if path.isfile(path.join(wals_dir, f))]
+		wals.sort()
+		with open(path.join(wals_dir, wals[-3]), "rb+") as f:
+			f.seek(256)
+			f.write(six.b("blablabla"))
+
+		id_backup = self.show_pb(node)[0].id
+		res = self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid])
+		self.assertIn(six.b("incorrect resource manager data checksum in record"), res)