tests: minor fixes, new tests for ptrack

tests/backup_test.py

@@ -162,7 +162,7 @@ class BackupTest(ProbackupTest, unittest.TestCase):
         except ProbackupException as e:
             self.assertTrue("INFO: Validate backups of the instance 'node'\n" in e.message
                 and 'WARNING: Backup file "{0}" is not found\n'.format(file) in e.message
-                and "WARNING: Backup {0} is corrupted\n".format(backup_id) in e.message
+                and "WARNING: Backup {0} data files are corrupted\n".format(backup_id) in e.message
                 and "INFO: Some backups are not valid\n" in e.message,
                 "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
 
@@ -231,32 +231,97 @@ class BackupTest(ProbackupTest, unittest.TestCase):
         self.del_test_dir(module_name, fname)
 
     # @unittest.skip("skip")
-    def test_page_checksumm_fail(self):
+    def test_checksumm_fail_heal_via_ptrack(self):
         """make node, corrupt some page, check that backup failed"""
         fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
             set_replication=True,
             initdb_params=['--data-checksums'],
             pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
             )
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         node.start()
 
-        self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"])
+        self.backup_node(
+            backup_dir, 'node', node,
+            backup_type="full", options=["-j", "4", "--stream"])
 
         node.safe_psql(
             "postgres",
-            "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,1000) i")
+            "create table t_heap as select 1 as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,1000) i")
         node.safe_psql(
             "postgres",
             "CHECKPOINT;")
 
-        heap_path = node.safe_psql("postgres", "select pg_relation_filepath('t_heap')").rstrip()
+        heap_path = node.safe_psql(
+            "postgres",
+            "select pg_relation_filepath('t_heap')").rstrip()
+
+        with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f:
+                f.seek(9000)
+                f.write(b"bla")
+                f.flush()
+                f.close
+
+        self.backup_node(
+            backup_dir, 'node', node, backup_type="full",
+            options=["-j", "4", "--stream", '--log-level-file=verbose'])
+
+        # open log file and check
+        with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f:
+            log_content = f.read()
+            self.assertIn('block 1, try fetching via SQL', log_content)
+            self.assertIn('SELECT pg_ptrack_get_block', log_content)
+            f.close
+
+        self.assertTrue(
+            self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK',
+            "Backup Status should be OK")
+
+        # Clean after yourself
+        # self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_checksumm_fail_heal_via_ptrack_fail(self):
+        """make node, corrupt some page, check that backup failed"""
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}
+        )
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        node.start()
+
+        self.backup_node(
+            backup_dir, 'node', node, backup_type="full",
+            options=["-j", "4", "--stream"])
+
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select 1 as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,1000) i")
+        node.safe_psql(
+            "postgres",
+            "CHECKPOINT;")
+
+        heap_path = node.safe_psql(
+            "postgres",
+            "select pg_relation_filepath('t_heap')").rstrip()
         node.stop()
 
-        with open(os.path.join(node.data_dir,heap_path), "rb+", 0) as f:
+        with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f:
                 f.seek(9000)
                 f.write(b"bla")
                 f.flush()
@@ -264,17 +329,27 @@ class BackupTest(ProbackupTest, unittest.TestCase):
         node.start()
 
         try:
-            self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"])
+            self.backup_node(
+                backup_dir, 'node', node,
+                backup_type="full", options=["-j", "4", "--stream"])
             # we should die here because exception is what we expect to happen
-            self.assertEqual(1, 0, "Expecting Error because of page corruption in PostgreSQL instance.\n Output: {0} \n CMD: {1}".format(
+            self.assertEqual(
-                repr(self.output), self.cmd))
+                1, 0,
+                "Expecting Error because of page "
+                "corruption in PostgreSQL instance.\n"
+                " Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertIn("ERROR: File", e.message,
+            self.assertTrue(
-                "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
+                "WARNING: File" in e.message and
-            self.assertIn("blknum", e.message,
+                "blknum" in e.message and
-                "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
+                "have wrong checksum" in e.message,
-            self.assertIn("have wrong checksum", e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(
-                "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
+                    repr(e.message), self.cmd))
+
+        self.assertTrue(
+             self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR',
+             "Backup Status should be ERROR")
 
         # Clean after yourself
-        self.del_test_dir(module_name, fname)
+        # self.del_test_dir(module_name, fname)

tests/expected/option_help.out

@@ -59,5 +59,15 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
   pg_probackup del-instance -B backup-dir
                  --instance=instance_name
 
+  pg_probackup archive-push -B backup-dir --instance=instance_name
+                 --wal-file-path=wal-file-path
+                 --wal-file-name=wal-file-name
+                 [--compress [--compress-level=compress-level]]
+                 [--overwrite]
+
+  pg_probackup archive-get -B backup-dir --instance=instance_name
+                 --wal-file-path=wal-file-path
+                 --wal-file-name=wal-file-name
+
 Read the website for details. <https://github.com/postgrespro/pg_probackup>
 Report bugs to <https://github.com/postgrespro/pg_probackup/issues>.

tests/helpers/ptrack_helpers.py

@@ -858,7 +858,7 @@ class ProbackupTest(object):
             'postmaster.pid', 'postmaster.opts',
             'pg_internal.init', 'postgresql.auto.conf',
             'backup_label', 'tablespace_map', 'recovery.conf',
-            'ptrack_control', 'ptrack_init'
+            'ptrack_control', 'ptrack_init', 'pg_control'
         ]
         suffixes_to_ignore = (
             '_ptrack'
@@ -1119,7 +1119,10 @@ class GDBobj(ProbackupTest):
                 continue
             if line.startswith('*stopped,reason="breakpoint-hit"'):
                 continue
-            if line.startswith('*stopped,reason="exited-normally"'):
+            if (
+                line.startswith('*stopped,reason="exited-normally"') or
+                line == '*stopped\n'
+            ):
                 return
         raise GdbException(
             'Failed to continue execution until exit.\n'
@@ -1164,11 +1167,10 @@ class GDBobj(ProbackupTest):
         self.proc.stdin.flush()
 
         while True:
-#            sleep(1)
             line = self.proc.stdout.readline()
             output += [line]
             if self.verbose:
-                print(line)
+                print(repr(line))
             if line == '^done\n' or line.startswith('*stopped'):
                 break
             if running and line.startswith('*running'):

tests/page.py

@@ -96,7 +96,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase):
         node_restored.start()
 
         # Clean after yourself
-        # self.del_test_dir(module_name, fname)
+        self.del_test_dir(module_name, fname)
 
     # @unittest.skip("skip")
     def test_page_stream(self):

tests/retention_test.py

@@ -125,7 +125,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
         self.backup_node(backup_dir, 'node', node)
         node.safe_psql(
             "postgres",
-            "insert into t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i")
+            "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i")
 
         self.backup_node(backup_dir, 'node', node)