db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge master into pgpro-2065

Browse Source
This commit is contained in:
Anastasia
2018-12-13 16:44:46 +03:00
parent 5b2cae5e81 e16bdc6660
commit 1654eea349
12 changed files with 380 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/catalog.c
-14
View File
@@ -820,20 +820,6 @@ pgBackupInit(pgBackup *backup)
backup->server_version[0] = '\0';
}
/*
* Copy backup metadata from **src** into **dst**.
*/
void
pgBackupCopy(pgBackup *dst, pgBackup *src)
{
pfree(dst->primary_conninfo);
memcpy(dst, src, sizeof(pgBackup));
if (src->primary_conninfo)
dst->primary_conninfo = pstrdup(src->primary_conninfo);
}
/* free pgBackup object */
void
pgBackupFree(void *backup)
src/data.c
+18 -7
View File
@@ -635,7 +635,7 @@ backup_data_file(backup_files_arg* arguments,
if (file->size % BLCKSZ != 0)
{
fclose(in);
elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size);
elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size);
}
/*
@@ -890,6 +890,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
if (write_header)
{
/* We uncompressed the page, so its size is BLCKSZ */
header.compressed_size = BLCKSZ;
if (fwrite(&header, 1, sizeof(header), out) != sizeof(header))
elog(ERROR, "cannot write header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno));
@@ -1653,19 +1655,23 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (read_len == 0 && feof(in))
break; /* EOF found */
else if (read_len != 0 && feof(in))
elog(ERROR,
elog(WARNING,
"odd size page found at block %u of \"%s\"",
blknum, file->path);
else
elog(ERROR, "cannot read header of block %u of \"%s\": %s",
elog(WARNING, "cannot read header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno_tmp));
return false;
}
COMP_FILE_CRC32(use_crc32c, crc, &header, read_len);
if (header.block < blknum)
elog(ERROR, "backup is broken at file->path %s block %u",
{
elog(WARNING, "backup is broken at file->path %s block %u",
file->path, blknum);
return false;
}
blknum = header.block;
@@ -1681,8 +1687,11 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
read_len = fread(compressed_page.data, 1,
MAXALIGN(header.compressed_size), in);
if (read_len != MAXALIGN(header.compressed_size))
elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d",
{
elog(WARNING, "cannot read block %u of \"%s\" read %zu of %d",
blknum, file->path, read_len, header.compressed_size);
return false;
}
COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len);
@@ -1709,11 +1718,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
is_valid = false;
continue;
}
elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
elog(WARNING, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
file->path, uncompressed_size);
return false;
}
if (validate_one_page(page.data, file, blknum,
stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID)
stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID)
is_valid = false;
}
else
src/dir.c
+1 -1
View File
@@ -211,7 +211,7 @@ pgFileInit(const char *path)
strcpy(file->path, path); /* enough buffer size guaranteed */
/* Get file name from the path */
file_name = strrchr(file->path, '/');
file_name = last_dir_separator(file->path);
if (file_name == NULL)
file->name = file->path;
else
src/merge.c
+22 -1
View File
@@ -91,7 +91,8 @@ do_merge(time_t backup_id)
}
else
{
Assert(dest_backup);
if (dest_backup == NULL)
elog(ERROR, "Target backup %s was not found", base36enc(backup_id));
if (backup->start_time != prev_parent)
continue;
@@ -335,6 +336,20 @@ delete_source_backup:
}
}
/*
* Rename FULL backup directory.
*/
elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id);
if (rename(to_backup_path, from_backup_path) == -1)
elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s",
to_backup_path, from_backup_path, strerror(errno));
/*
* Merging finished, now we can safely update ID of the destination backup.
*/
to_backup->start_time = from_backup->start_time;
write_backup(to_backup);
/* Cleanup */
if (threads)
{
@@ -526,6 +541,12 @@ merge_files(void *arg)
else
copy_file(argument->from_root, argument->to_root, file);
/*
* We need to save compression algorithm type of the target backup to be
* able to restore in the future.
*/
file->compress_alg = to_backup->compress_alg;
if (file->write_size != BYTES_INVALID)
elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes",
file->path, file->write_size);
src/pg_probackup.c
-1
View File
@@ -345,7 +345,6 @@ main(int argc, char *argv[])
elog(ERROR, "-B, --backup-path must be a path to directory");
}
/* TODO it would be better to list commands that require instance option */
/* Option --instance is required for all commands except init and show */
if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD &&
backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD)
src/pg_probackup.h
-1
View File
@@ -487,7 +487,6 @@ extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len,
const char *subdir1, const char *subdir2);
extern int pgBackupCreateDir(pgBackup *backup);
extern void pgBackupInit(pgBackup *backup);
extern void pgBackupCopy(pgBackup *dst, pgBackup *src);
extern void pgBackupFree(void *backup);
extern int pgBackupCompareId(const void *f1, const void *f2);
extern int pgBackupCompareIdDesc(const void *f1, const void *f2);
src/utils/logger.c
+3 -3
View File
@@ -78,9 +78,9 @@ init_logger(const char *root_path, LoggerConfig *config)
if (config->log_level_file != LOG_OFF
&& config->log_directory == NULL)
{
config->log_directory = pgut_malloc(MAXPGPATH);
join_path_components(config->log_directory,
root_path, LOG_DIRECTORY_DEFAULT);
config->log_directory = pgut_malloc(MAXPGPATH);
join_path_components(config->log_directory,
root_path, LOG_DIRECTORY_DEFAULT);
}
logger_config = *config;
src/utils/pgut.c
+4 -1
View File
@@ -703,7 +703,10 @@ on_interrupt(void)
/* Set interruped flag */
interrupted = true;
/* User promts password, call on_cleanup() byhand */
/*
* User promts password, call on_cleanup() byhand. Unless we do that we will
* get stuck forever until a user enters a password.
*/
if (in_password)
{
on_cleanup();
src/validate.c
+8
View File
@@ -52,6 +52,14 @@ pgBackupValidate(pgBackup *backup)
validate_files_arg *threads_args;
int i;
/* Check backup version */
if (backup->program_version &&
parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION))
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
"pg_probackup do not guarantee to be forward compatible. "
"Please upgrade pg_probackup binary.",
PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version);
/* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */
if (backup->status != BACKUP_STATUS_OK &&
backup->status != BACKUP_STATUS_DONE &&
tests/helpers/ptrack_helpers.py
+44 -4
View File
@@ -1064,7 +1064,7 @@ class ProbackupTest(object):
except:
pass
def pgdata_content(self, directory, ignore_ptrack=True):
def pgdata_content(self, pgdata, ignore_ptrack=True):
""" return dict with directory content. "
" TAKE IT AFTER CHECKPOINT or BACKUP"""
dirs_to_ignore = [
@@ -1081,9 +1081,10 @@ class ProbackupTest(object):
# '_ptrack'
# )
directory_dict = {}
directory_dict['pgdata'] = directory
directory_dict['pgdata'] = pgdata
directory_dict['files'] = {}
for root, dirs, files in os.walk(directory, followlinks=True):
directory_dict['dirs'] = []
for root, dirs, files in os.walk(pgdata, followlinks=True):
dirs[:] = [d for d in dirs if d not in dirs_to_ignore]
for file in files:
if (
@@ -1093,7 +1094,7 @@ class ProbackupTest(object):
continue
file_fullpath = os.path.join(root, file)
file_relpath = os.path.relpath(file_fullpath, directory)
file_relpath = os.path.relpath(file_fullpath, pgdata)
directory_dict['files'][file_relpath] = {'is_datafile': False}
directory_dict['files'][file_relpath]['md5'] = hashlib.md5(
open(file_fullpath, 'rb').read()).hexdigest()
@@ -1106,12 +1107,51 @@ class ProbackupTest(object):
file_fullpath, size_in_pages
)
for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True):
for directory in dirs:
directory_path = os.path.join(root, directory)
directory_relpath = os.path.relpath(directory_path, pgdata)
found = False
for d in dirs_to_ignore:
if d in directory_relpath:
found = True
break
# check if directory already here as part of larger directory
if not found:
for d in directory_dict['dirs']:
# print("OLD dir {0}".format(d))
if directory_relpath in d:
found = True
break
if not found:
directory_dict['dirs'].append(directory_relpath)
return directory_dict
def compare_pgdata(self, original_pgdata, restored_pgdata):
""" return dict with directory content. DO IT BEFORE RECOVERY"""
fail = False
error_message = 'Restored PGDATA is not equal to original!\n'
# Compare directories
for directory in restored_pgdata['dirs']:
if directory not in original_pgdata['dirs']:
fail = True
error_message += '\nDirectory was not present'
error_message += ' in original PGDATA: {0}\n'.format(
os.path.join(restored_pgdata['pgdata'], directory))
for directory in original_pgdata['dirs']:
if directory not in restored_pgdata['dirs']:
fail = True
error_message += '\nDirectory dissappeared'
error_message += ' in restored PGDATA: {0}\n'.format(
os.path.join(restored_pgdata['pgdata'], directory))
for file in restored_pgdata['files']:
# File is present in RESTORED PGDATA
# but not present in ORIGINAL
tests/merge.py
+162 -1
View File
@@ -3,6 +3,7 @@
import unittest
import os
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
import shutil
module_name = "merge"
@@ -472,8 +473,167 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
def test_merge_tablespaces(self):
"""
Some test here
Create tablespace with table, take FULL backup,
create another tablespace with another table and drop previous
tablespace, take page backup, merge it and restore
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, fname),
set_replication=True, initdb_params=['--data-checksums'],
pg_options={
'wal_level': 'replica',
'max_wal_senders': '2',
'autovacuum': 'off'
}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
node.safe_psql(
"postgres",
"create table t_heap tablespace somedata as select i as id,"
" md5(i::text) as text, md5(i::text)::tsvector as tsvector"
" from generate_series(0,100) i"
)
# FULL backup
self.backup_node(backup_dir, 'node', node)
# Create new tablespace
self.create_tblspace_in_node(node, 'somedata1')
node.safe_psql(
"postgres",
"create table t_heap1 tablespace somedata1 as select i as id,"
" md5(i::text) as text, md5(i::text)::tsvector as tsvector"
" from generate_series(0,100) i"
)
node.safe_psql(
"postgres",
"drop table t_heap"
)
# Drop old tablespace
node.safe_psql(
"postgres",
"drop tablespace somedata"
)
# PAGE backup
backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page")
pgdata = self.pgdata_content(node.data_dir)
node.stop()
shutil.rmtree(
self.get_tblspace_path(node, 'somedata'),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(node, 'somedata1'),
ignore_errors=True)
node.cleanup()
self.merge_backup(backup_dir, 'node', backup_id)
self.restore_node(
backup_dir, 'node', node, options=["-j", "4"])
pgdata_restored = self.pgdata_content(node.data_dir)
# this compare should fall because we lost some directories
self.compare_pgdata(pgdata, pgdata_restored)
# @unittest.skip("skip")
def test_merge_tablespaces_1(self):
"""
Create tablespace with table, take FULL backup,
create another tablespace with another table, take page backup,
drop first tablespace and take delta backup,
merge it and restore
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, fname),
set_replication=True, initdb_params=['--data-checksums'],
pg_options={
'wal_level': 'replica',
'max_wal_senders': '2',
'autovacuum': 'off'
}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
self.create_tblspace_in_node(node, 'somedata')
# FULL backup
self.backup_node(backup_dir, 'node', node)
node.safe_psql(
"postgres",
"create table t_heap tablespace somedata as select i as id,"
" md5(i::text) as text, md5(i::text)::tsvector as tsvector"
" from generate_series(0,100) i"
)
# CREATE NEW TABLESPACE
self.create_tblspace_in_node(node, 'somedata1')
node.safe_psql(
"postgres",
"create table t_heap1 tablespace somedata1 as select i as id,"
" md5(i::text) as text, md5(i::text)::tsvector as tsvector"
" from generate_series(0,100) i"
)
# PAGE backup
self.backup_node(backup_dir, 'node', node, backup_type="page")
node.safe_psql(
"postgres",
"drop table t_heap"
)
node.safe_psql(
"postgres",
"drop tablespace somedata"
)
# DELTA backup
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type="delta")
pgdata = self.pgdata_content(node.data_dir)
node.stop()
shutil.rmtree(
self.get_tblspace_path(node, 'somedata'),
ignore_errors=True)
shutil.rmtree(
self.get_tblspace_path(node, 'somedata1'),
ignore_errors=True)
node.cleanup()
self.merge_backup(backup_dir, 'node', backup_id)
self.restore_node(
backup_dir, 'node', node,
options=["-j", "4"])
pgdata_restored = self.pgdata_content(node.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)
def test_merge_page_truncate(self):
"""
@@ -1048,3 +1208,4 @@ class MergeTest(ProbackupTest, unittest.TestCase):
# FULL MERGING
# 3. Need new test with corrupted FULL backup
# 4. different compression levels
tests/page.py
+118
View File
@@ -1,6 +1,7 @@
import os
import unittest
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
from testgres import QueryException
from datetime import datetime, timedelta
import subprocess
import gzip
@@ -1030,3 +1031,120 @@ class PageBackupTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_page_create_db(self):
"""
Make node, take full backup, create database db1, take page backup,
restore database and check it presense
"""
self.maxDiff = None
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, fname),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={
'max_wal_size': '10GB',
'max_wal_senders': '2',
'checkpoint_timeout': '5min',
'autovacuum': 'off'
}
)
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
# FULL BACKUP
node.safe_psql(
"postgres",
"create table t_heap as select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
self.backup_node(
backup_dir, 'node', node)
# CREATE DATABASE DB1
node.safe_psql("postgres", "create database db1")
node.safe_psql(
"db1",
"create table t_heap as select i as id, md5(i::text) as text, "
"md5(i::text)::tsvector as tsvector from generate_series(0,1000) i")
# PAGE BACKUP
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
# RESTORE
node_restored = self.make_simple_node(
base_dir="{0}/{1}/node_restored".format(module_name, fname)
)
node_restored.cleanup()
self.restore_node(
backup_dir, 'node', node_restored,
backup_id=backup_id, options=["-j", "4"])
# COMPARE PHYSICAL CONTENT
if self.paranoia:
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# START RESTORED NODE
node_restored.append_conf(
"postgresql.auto.conf", "port = {0}".format(node_restored.port))
node_restored.slow_start()
node_restored.safe_psql('db1', 'select 1')
node_restored.cleanup()
# DROP DATABASE DB1
node.safe_psql(
"postgres", "drop database db1")
# SECOND PTRACK BACKUP
backup_id = self.backup_node(
backup_dir, 'node', node, backup_type='page')
if self.paranoia:
pgdata = self.pgdata_content(node.data_dir)
# RESTORE SECOND PTRACK BACKUP
self.restore_node(
backup_dir, 'node', node_restored,
backup_id=backup_id, options=["-j", "4"]
)
# COMPARE PHYSICAL CONTENT
if self.paranoia:
pgdata_restored = self.pgdata_content(
node_restored.data_dir, ignore_ptrack=False)
self.compare_pgdata(pgdata, pgdata_restored)
# START RESTORED NODE
node_restored.append_conf(
"postgresql.auto.conf", "port = {0}".format(node_restored.port))
node_restored.slow_start()
try:
node_restored.safe_psql('db1', 'select 1')
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because we are connecting to deleted database"
"\n Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd)
)
except QueryException as e:
self.assertTrue(
'FATAL: database "db1" does not exist' in e.message,
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
repr(e.message), self.cmd)
)
# Clean after yourself
self.del_test_dir(module_name, fname)