db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-24 08:52:38 +02:00
Code Issues Releases Activity

Merge branch 'master' into release_2_4

Browse Source
This commit is contained in:
Grigory Smolkin 2020-06-08 20:05:46 +03:00
commit 08b9cb7689
9 changed files with 159 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
README.md
View File

@ -54,7 +54,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
## Current release
[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4)
[2.3.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.5)
## Documentation
@ -62,7 +62,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku
## Installation and Setup
### Windows Installation
Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.3).
Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.5).
### Linux Installation
#### pg_probackup for vanilla PostgreSQL
@ -96,64 +96,64 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
#SRPM Packages
yumdownloader --source pg_probackup-{12,11,10,9.6,9.5}
#RPM ALT Linux p7
#RPM ALT Linux 7
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
sudo apt-get update
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
#RPM ALT Linux p8
#RPM ALT Linux 8
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
sudo apt-get update
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
#RPM ALT Linux p9
#RPM ALT Linux 9
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
sudo apt-get update
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
```
#### pg_probackup for PostgresPro Standart and Enterprise
#### pg_probackup for PostgresPro Standard and Enterprise
```shell
#DEB Ubuntu|Debian Packages
sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list
sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update
sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}
sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg
sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}
sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg
#RPM Centos Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm
yum install pg_probackup-{std,ent}-{11,10,9.6}
yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
yum install pg_probackup-{std,ent}-{12,11,10,9.6}
yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
#RPM RHEL Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm
yum install pg_probackup-{std,ent}-{11,10,9.6}
yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
yum install pg_probackup-{std,ent}-{12,11,10,9.6}
yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
#RPM Oracle Linux Packages
rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm
yum install pg_probackup-{std,ent}-{11,10,9.6}
yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
yum install pg_probackup-{std,ent}-{12,11,10,9.6}
yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
#RPM ALT Linux p7
#RPM ALT Linux 7
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list
sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
#RPM ALT Linux p8
#RPM ALT Linux 8
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list
sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
#RPM ALT Linux p9
#RPM ALT Linux 9
sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
```
Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup).

2
src/backup.c
View File

@ -1855,7 +1855,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
}
elog(LOG, "stop_lsn: %X/%X",
(uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));
(uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
/* Write backup_label and tablespace_map */
if (!exclusive_backup)

17
src/init.c
View File

@ -67,20 +67,19 @@ do_add_instance(InstanceConfig *instance)
/* Ensure that all root directories already exist */
if (access(backup_path, F_OK) != 0)
elog(ERROR, "%s directory does not exist.", backup_path);
elog(ERROR, "Directory does not exist: '%s'", backup_path);
join_path_components(path, backup_path, BACKUPS_DIR);
if (access(path, F_OK) != 0)
elog(ERROR, "%s directory does not exist.", path);
elog(ERROR, "Directory does not exist: '%s'", path);
join_path_components(arclog_path_dir, backup_path, "wal");
if (access(arclog_path_dir, F_OK) != 0)
elog(ERROR, "%s directory does not exist.", arclog_path_dir);
elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir);
/* Create directory for data files of this specific instance */
if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode))
elog(ERROR, "instance '%s' already exists", instance->backup_instance_path);
dir_create_dir(instance->backup_instance_path, DIR_PERMISSION);
elog(ERROR, "Instance '%s' backup directory already exists: '%s'",
instance->name, instance->backup_instance_path);
/*
* Create directory for wal files of this specific instance.
@ -88,7 +87,11 @@ do_add_instance(InstanceConfig *instance)
* directory in data dir, we shouldn't have it in wal as well.
*/
if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode))
elog(ERROR, "arclog_path '%s' already exists", instance->arclog_path);
elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'",
instance->name, instance->arclog_path);
/* Create directory for data files of this specific instance */
dir_create_dir(instance->backup_instance_path, DIR_PERMISSION);
dir_create_dir(instance->arclog_path, DIR_PERMISSION);
/*

4
src/pg_probackup.h
View File

@ -241,8 +241,8 @@ typedef enum ShowFormat
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.3.4"
#define AGENT_PROTOCOL_VERSION 20304
#define PROGRAM_VERSION "2.3.5"
#define AGENT_PROTOCOL_VERSION 20305
typedef struct ConnectionOptions

8
src/validate.c
View File

@ -58,13 +58,19 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
int i;
// parray *dbOid_exclude_list = NULL;
/* Check backup version */
/* Check backup program version */
if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION))
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
"pg_probackup do not guarantee to be forward compatible. "
"Please upgrade pg_probackup binary.",
PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version);
/* Check backup server version */
if (strcmp(backup->server_version, PG_MAJORVERSION) != 0)
elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary "
"compiled with server version %s",
base36enc(backup->start_time), backup->server_version, PG_MAJORVERSION);
if (backup->status == BACKUP_STATUS_RUNNING)
{
elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation",

2
tests/expected/option_version.out
View File

@ -1 +1 @@
pg_probackup 2.3.4
pg_probackup 2.3.5

51
tests/init.py
View File

@ -1,6 +1,7 @@
import os
import unittest
from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException
import shutil
module_name = 'init'
@ -104,3 +105,53 @@ class InitTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_add_instance_idempotence(self):
"""
https://github.com/postgrespro/pg_probackup/issues/219
"""
fname = self.id().split(".")[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'))
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
shutil.rmtree(os.path.join(backup_dir, 'backups', 'node'))
dir_backups = os.path.join(backup_dir, 'backups', 'node')
dir_wal = os.path.join(backup_dir, 'wal', 'node')
try:
self.add_instance(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because page backup should not be possible "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Instance 'node' WAL archive directory already exists: ",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
try:
self.add_instance(backup_dir, 'node', node)
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because page backup should not be possible "
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Instance 'node' WAL archive directory already exists: ",
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)

18
tests/merge.py
View File

@ -3,6 +3,7 @@
import unittest
import os
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import QueryException
import shutil
from datetime import datetime, timedelta
import time
@ -175,10 +176,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True, initdb_params=["--data-checksums"],
pg_options={
'autovacuum': 'off'
}
)
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, "node", node)
@ -186,7 +184,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
node.slow_start()
# Fill with data
node.pgbench_init(scale=5)
node.pgbench_init(scale=10)
# Do compressed FULL backup
self.backup_node(backup_dir, "node", node, options=['--compress', '--stream'])
@ -196,7 +194,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
self.assertEqual(show_backup["backup-mode"], "FULL")
# Change data
pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum'])
pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
pgbench.wait()
# Do compressed DELTA backup
@ -205,7 +203,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
backup_type="delta", options=['--compress', '--stream'])
# Change data
pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum'])
pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
pgbench.wait()
# Do compressed PAGE backup
@ -1478,10 +1476,12 @@ class MergeTest(ProbackupTest, unittest.TestCase):
self.del_test_dir(module_name, fname)
@unittest.skip("skip")
def test_crash_after_opening_backup_control_2(self):
"""
check that crashing after opening backup_content.control
for writing will not result in losing metadata about backup files
TODO: rewrite
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@ -1531,7 +1531,7 @@ class MergeTest(ProbackupTest, unittest.TestCase):
gdb.run_until_break()
gdb.set_breakpoint('sprintf')
gdb.continue_execution_until_break(20)
gdb.continue_execution_until_break(1)
gdb._execute('signal SIGKILL')
@ -1568,10 +1568,12 @@ class MergeTest(ProbackupTest, unittest.TestCase):
self.del_test_dir(module_name, fname)
@unittest.skip("skip")
def test_losing_file_after_failed_merge(self):
"""
check that crashing after opening backup_content.control
for writing will not result in losing metadata about backup files
TODO: rewrite
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

55
tests/validate.py
View File

@ -3716,6 +3716,59 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_not_validate_diffenent_pg_version(self):
"""Do not validate backup, if binary is compiled with different PG version"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
initdb_params=['--data-checksums'])
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
backup_id = self.backup_node(backup_dir, 'node', node)
control_file = os.path.join(
backup_dir, "backups", "node", backup_id,
"backup.control")
pg_version = node.major_version
if pg_version.is_integer():
pg_version = int(pg_version)
fake_new_pg_version = pg_version + 1
with open(control_file, 'r') as f:
data = f.read();
data = data.replace(str(pg_version), str(fake_new_pg_version))
with open(control_file, 'w') as f:
f.write(data);
try:
self.validate_pb(backup_dir)
self.assertEqual(
1, 0,
"Expecting Error because validation is forbidden if server version of backup "
"is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertIn(
"ERROR: Backup {0} has server version".format(backup_id),
e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
# Clean after yourself
self.del_test_dir(module_name, fname)
# validate empty backup list
# page from future during validate
# page from future during backup
@ -3727,4 +3780,4 @@ class ValidateTest(ProbackupTest, unittest.TestCase):
# 715 MAXALIGN(header.compressed_size), in);
# 716 if (read_len != MAXALIGN(header.compressed_size))
# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d",
# 718 blknum, file->path, read_len, header.compressed_size);
# 718 blknum, file->path, read_len, header.compressed_size);