mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2024-12-03 09:59:53 +02:00
Merge branch 'master' into pgpro-2071
This commit is contained in:
Arthur Zakirov
commit
b4e707b4d1
25
src/backup.c
25
src/backup.c
@ -24,8 +24,6 @@
|
||||
#include "utils/thread.h"
|
||||
#include <time.h>
|
||||
|
||||
#define PG_STOP_BACKUP_TIMEOUT 300
|
||||
|
||||
/*
|
||||
* Macro needed to parse ptrack.
|
||||
* NOTE Keep those values syncronised with definitions in ptrack.h
|
||||
@ -310,7 +308,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
|
||||
to_path, strerror(errno_tmp));
|
||||
}
|
||||
|
||||
INIT_TRADITIONAL_CRC32(file->crc);
|
||||
INIT_FILE_CRC32(true, file->crc);
|
||||
|
||||
/* read from stream and write to backup file */
|
||||
while (1)
|
||||
@ -336,14 +334,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
|
||||
{
|
||||
write_buffer_size = Min(row_length, sizeof(buf));
|
||||
memcpy(buf, copybuf, write_buffer_size);
|
||||
COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size);
|
||||
COMP_FILE_CRC32(true, file->crc, buf, write_buffer_size);
|
||||
|
||||
/* TODO calc checksum*/
|
||||
if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
|
||||
{
|
||||
errno_tmp = errno;
|
||||
/* oops */
|
||||
FIN_TRADITIONAL_CRC32(file->crc);
|
||||
FIN_FILE_CRC32(true, file->crc);
|
||||
fclose(out);
|
||||
PQfinish(conn);
|
||||
elog(ERROR, "cannot write to \"%s\": %s", to_path,
|
||||
@ -367,7 +365,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
|
||||
}
|
||||
|
||||
file->write_size = (int64) file->read_size;
|
||||
FIN_TRADITIONAL_CRC32(file->crc);
|
||||
FIN_FILE_CRC32(true, file->crc);
|
||||
|
||||
fclose(out);
|
||||
}
|
||||
@ -1883,8 +1881,8 @@ pg_stop_backup(pgBackup *backup)
|
||||
}
|
||||
|
||||
/*
|
||||
* Wait for the result of pg_stop_backup(),
|
||||
* but no longer than PG_STOP_BACKUP_TIMEOUT seconds
|
||||
* Wait for the result of pg_stop_backup(), but no longer than
|
||||
* archive_timeout seconds
|
||||
*/
|
||||
if (pg_stop_backup_is_sent && !in_cleanup)
|
||||
{
|
||||
@ -1907,14 +1905,14 @@ pg_stop_backup(pgBackup *backup)
|
||||
elog(INFO, "wait for pg_stop_backup()");
|
||||
|
||||
/*
|
||||
* If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds,
|
||||
* If postgres haven't answered in archive_timeout seconds,
|
||||
* send an interrupt.
|
||||
*/
|
||||
if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT)
|
||||
if (pg_stop_backup_timeout > instance_config.archive_timeout)
|
||||
{
|
||||
pgut_cancel(conn);
|
||||
elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it",
|
||||
PG_STOP_BACKUP_TIMEOUT);
|
||||
instance_config.archive_timeout);
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -2290,9 +2288,12 @@ backup_files(void *arg)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
else if (strcmp(file->name, "pg_control") == 0)
|
||||
copy_pgcontrol_file(arguments->from_root, arguments->to_root,
|
||||
file);
|
||||
else
|
||||
{
|
||||
bool skip = false;
|
||||
bool skip = false;
|
||||
|
||||
/* If non-data file has not changed since last backup... */
|
||||
if (prev_file && file->exists_in_prev &&
|
||||
|
||||
26
src/data.c
26
src/data.c
@ -487,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
|
||||
blknum, header.compressed_size, write_buffer_size); */
|
||||
|
||||
/* Update CRC */
|
||||
COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
|
||||
COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size);
|
||||
|
||||
/* write data page */
|
||||
if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
|
||||
@ -547,13 +547,13 @@ backup_data_file(backup_files_arg* arguments,
|
||||
/* reset size summary */
|
||||
file->read_size = 0;
|
||||
file->write_size = 0;
|
||||
INIT_TRADITIONAL_CRC32(file->crc);
|
||||
INIT_FILE_CRC32(true, file->crc);
|
||||
|
||||
/* open backup mode file for read */
|
||||
in = fopen(file->path, PG_BINARY_R);
|
||||
if (in == NULL)
|
||||
{
|
||||
FIN_TRADITIONAL_CRC32(file->crc);
|
||||
FIN_FILE_CRC32(true, file->crc);
|
||||
|
||||
/*
|
||||
* If file is not found, this is not en error.
|
||||
@ -658,7 +658,7 @@ backup_data_file(backup_files_arg* arguments,
|
||||
to_path, strerror(errno));
|
||||
fclose(in);
|
||||
|
||||
FIN_TRADITIONAL_CRC32(file->crc);
|
||||
FIN_FILE_CRC32(true, file->crc);
|
||||
|
||||
/*
|
||||
* If we have pagemap then file in the backup can't be a zero size.
|
||||
@ -927,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
|
||||
struct stat st;
|
||||
pg_crc32 crc;
|
||||
|
||||
INIT_TRADITIONAL_CRC32(crc);
|
||||
INIT_FILE_CRC32(true, crc);
|
||||
|
||||
/* reset size summary */
|
||||
file->read_size = 0;
|
||||
@ -937,7 +937,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
|
||||
in = fopen(file->path, PG_BINARY_R);
|
||||
if (in == NULL)
|
||||
{
|
||||
FIN_TRADITIONAL_CRC32(crc);
|
||||
FIN_FILE_CRC32(true, crc);
|
||||
file->crc = crc;
|
||||
|
||||
/* maybe deleted, it's not error */
|
||||
@ -986,7 +986,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
|
||||
strerror(errno_tmp));
|
||||
}
|
||||
/* update CRC */
|
||||
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
|
||||
COMP_FILE_CRC32(true, crc, buf, read_len);
|
||||
|
||||
file->read_size += read_len;
|
||||
}
|
||||
@ -1013,14 +1013,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
|
||||
strerror(errno_tmp));
|
||||
}
|
||||
/* update CRC */
|
||||
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
|
||||
COMP_FILE_CRC32(true, crc, buf, read_len);
|
||||
|
||||
file->read_size += read_len;
|
||||
}
|
||||
|
||||
file->write_size = (int64) file->read_size;
|
||||
/* finish CRC calculation and store into pgFile */
|
||||
FIN_TRADITIONAL_CRC32(crc);
|
||||
FIN_FILE_CRC32(true, crc);
|
||||
file->crc = crc;
|
||||
|
||||
/* update file permission */
|
||||
@ -1424,7 +1424,7 @@ calc_file_checksum(pgFile *file)
|
||||
{
|
||||
Assert(S_ISREG(file->mode));
|
||||
|
||||
file->crc = pgFileGetCRC(file->path, false, false, &file->read_size);
|
||||
file->crc = pgFileGetCRC(file->path, true, false, &file->read_size);
|
||||
file->write_size = file->read_size;
|
||||
}
|
||||
|
||||
@ -1547,14 +1547,14 @@ validate_one_page(Page page, pgFile *file,
|
||||
|
||||
/* Valiate pages of datafile in backup one by one */
|
||||
bool
|
||||
check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
|
||||
uint32 checksum_version, uint32 backup_version)
|
||||
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
uint32 backup_version)
|
||||
{
|
||||
size_t read_len = 0;
|
||||
bool is_valid = true;
|
||||
FILE *in;
|
||||
pg_crc32 crc;
|
||||
bool use_crc32c = (backup_version <= 20021);
|
||||
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
|
||||
|
||||
elog(VERBOSE, "validate relation blocks for file %s", file->name);
|
||||
|
||||
|
||||
@ -524,9 +524,11 @@ merge_files(void *arg)
|
||||
* do that.
|
||||
*/
|
||||
file->write_size = pgFileSize(to_path_tmp);
|
||||
file->crc = pgFileGetCRC(to_path_tmp, false, true, NULL);
|
||||
file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL);
|
||||
}
|
||||
}
|
||||
else if (strcmp(file->name, "pg_control") == 0)
|
||||
copy_pgcontrol_file(argument->from_root, argument->to_root, file);
|
||||
else
|
||||
copy_file(argument->from_root, argument->to_root, file);
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
#include "utils/thread.h"
|
||||
#include <time.h>
|
||||
|
||||
const char *PROGRAM_VERSION = "2.0.24";
|
||||
const char *PROGRAM_VERSION = "2.0.25";
|
||||
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
|
||||
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";
|
||||
|
||||
|
||||
@ -96,6 +96,7 @@ do { \
|
||||
FIN_TRADITIONAL_CRC32(crc); \
|
||||
} while (0)
|
||||
|
||||
|
||||
/* Information about single file (or dir) in backup */
|
||||
typedef struct pgFile
|
||||
{
|
||||
@ -539,8 +540,7 @@ extern void get_wal_file(const char *from_path, const char *to_path);
|
||||
|
||||
extern void calc_file_checksum(pgFile *file);
|
||||
|
||||
extern bool check_file_pages(pgFile* file,
|
||||
XLogRecPtr stop_lsn,
|
||||
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
|
||||
uint32 checksum_version, uint32 backup_version);
|
||||
/* parsexlog.c */
|
||||
extern void extractPageMap(const char *archivedir,
|
||||
@ -567,11 +567,15 @@ extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn,
|
||||
/* in util.c */
|
||||
extern TimeLineID get_current_timeline(bool safe);
|
||||
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
|
||||
extern uint64 get_system_identifier(char *pgdata);
|
||||
extern uint64 get_system_identifier(const char *pgdata_path);
|
||||
extern uint64 get_remote_system_identifier(PGconn *conn);
|
||||
extern uint32 get_data_checksum_version(bool safe);
|
||||
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
|
||||
extern uint32 get_xlog_seg_size(char *pgdata_path);
|
||||
extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn);
|
||||
extern void set_min_recovery_point(pgFile *file, const char *backup_path,
|
||||
XLogRecPtr stop_backup_lsn);
|
||||
extern void copy_pgcontrol_file(const char *from_root, const char *to_root,
|
||||
pgFile *file);
|
||||
|
||||
extern void sanityChecks(void);
|
||||
extern void time2iso(char *buf, size_t len, time_t time);
|
||||
|
||||
@ -632,6 +632,8 @@ restore_files(void *arg)
|
||||
false,
|
||||
parse_program_version(arguments->backup->program_version));
|
||||
}
|
||||
else if (strcmp(file->name, "pg_control") == 0)
|
||||
copy_pgcontrol_file(from_root, instance_config.pgdata, file);
|
||||
else
|
||||
copy_file(from_root, instance_config.pgdata, file);
|
||||
|
||||
|
||||
@ -363,7 +363,7 @@ show_instance_plain(parray *backup_list, bool show_name)
|
||||
time2iso(row->recovery_time, lengthof(row->recovery_time),
|
||||
backup->recovery_time);
|
||||
else
|
||||
StrNCpy(row->recovery_time, "----", 4);
|
||||
StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time));
|
||||
widths[cur] = Max(widths[cur], strlen(row->recovery_time));
|
||||
cur++;
|
||||
|
||||
@ -388,7 +388,7 @@ show_instance_plain(parray *backup_list, bool show_name)
|
||||
snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0,
|
||||
difftime(backup->end_time, backup->start_time));
|
||||
else
|
||||
StrNCpy(row->duration, "----", 4);
|
||||
StrNCpy(row->duration, "----", sizeof(row->duration));
|
||||
widths[cur] = Max(widths[cur], strlen(row->duration));
|
||||
cur++;
|
||||
|
||||
|
||||
57
src/util.c
57
src/util.c
@ -207,7 +207,7 @@ get_checkpoint_location(PGconn *conn)
|
||||
}
|
||||
|
||||
uint64
|
||||
get_system_identifier(char *pgdata_path)
|
||||
get_system_identifier(const char *pgdata_path)
|
||||
{
|
||||
ControlFileData ControlFile;
|
||||
char *buffer;
|
||||
@ -295,7 +295,27 @@ get_data_checksum_version(bool safe)
|
||||
return ControlFile.data_checksum_version;
|
||||
}
|
||||
|
||||
/* MinRecoveryPoint 'as-is' is not to be trusted */
|
||||
pg_crc32c
|
||||
get_pgcontrol_checksum(const char *pgdata_path)
|
||||
{
|
||||
ControlFileData ControlFile;
|
||||
char *buffer;
|
||||
size_t size;
|
||||
|
||||
/* First fetch file... */
|
||||
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false);
|
||||
if (buffer == NULL)
|
||||
return 0;
|
||||
digestControlFile(&ControlFile, buffer, size);
|
||||
pg_free(buffer);
|
||||
|
||||
return ControlFile.crc;
|
||||
}
|
||||
|
||||
/*
|
||||
* Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint
|
||||
* 'as-is' is not to be trusted.
|
||||
*/
|
||||
void
|
||||
set_min_recovery_point(pgFile *file, const char *backup_path,
|
||||
XLogRecPtr stop_backup_lsn)
|
||||
@ -324,20 +344,41 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
|
||||
|
||||
/* Update checksum in pg_control header */
|
||||
INIT_CRC32C(ControlFile.crc);
|
||||
COMP_CRC32C(ControlFile.crc,
|
||||
(char *) &ControlFile,
|
||||
COMP_CRC32C(ControlFile.crc, (char *) &ControlFile,
|
||||
offsetof(ControlFileData, crc));
|
||||
FIN_CRC32C(ControlFile.crc);
|
||||
|
||||
/* paranoia */
|
||||
checkControlFile(&ControlFile);
|
||||
|
||||
/* overwrite pg_control */
|
||||
snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE);
|
||||
writeControlFile(&ControlFile, fullpath);
|
||||
|
||||
/* Update pg_control checksum in backup_list */
|
||||
file->crc = pgFileGetCRC(fullpath, false, true, NULL);
|
||||
file->crc = ControlFile.crc;
|
||||
|
||||
pg_free(buffer);
|
||||
}
|
||||
|
||||
/*
|
||||
* Copy pg_control file to backup. We do not apply compression to this file.
|
||||
*/
|
||||
void
|
||||
copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file)
|
||||
{
|
||||
ControlFileData ControlFile;
|
||||
char *buffer;
|
||||
size_t size;
|
||||
char to_path[MAXPGPATH];
|
||||
|
||||
buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false);
|
||||
|
||||
digestControlFile(&ControlFile, buffer, size);
|
||||
|
||||
file->crc = ControlFile.crc;
|
||||
file->read_size = size;
|
||||
file->write_size = size;
|
||||
|
||||
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
|
||||
writeControlFile(&ControlFile, to_path);
|
||||
|
||||
pg_free(buffer);
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ static bool corrupted_backup_found = false;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
const char *base_path;
|
||||
parray *files;
|
||||
bool corrupted;
|
||||
XLogRecPtr stop_lsn;
|
||||
@ -101,6 +102,7 @@ pgBackupValidate(pgBackup *backup)
|
||||
{
|
||||
validate_files_arg *arg = &(threads_args[i]);
|
||||
|
||||
arg->base_path = base_path;
|
||||
arg->files = files;
|
||||
arg->corrupted = false;
|
||||
arg->stop_lsn = backup->stop_lsn;
|
||||
@ -223,9 +225,17 @@ pgBackupValidateFiles(void *arg)
|
||||
* CRC-32C algorithm.
|
||||
* To avoid this problem we need to use different algorithm, CRC-32 in
|
||||
* this case.
|
||||
*
|
||||
* Starting from 2.0.25 we calculate crc of pg_control differently.
|
||||
*/
|
||||
crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021,
|
||||
true, NULL);
|
||||
if (arguments->backup_version >= 20025 &&
|
||||
strcmp(file->name, "pg_control") == 0)
|
||||
crc = get_pgcontrol_checksum(arguments->base_path);
|
||||
else
|
||||
crc = pgFileGetCRC(file->path,
|
||||
arguments->backup_version <= 20021 ||
|
||||
arguments->backup_version >= 20025,
|
||||
true, NULL);
|
||||
if (crc != file->crc)
|
||||
{
|
||||
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
|
||||
|
||||
@ -2,7 +2,7 @@ import os
|
||||
import shutil
|
||||
import gzip
|
||||
import unittest
|
||||
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script
|
||||
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException
|
||||
from datetime import datetime, timedelta
|
||||
import subprocess
|
||||
from sys import exit
|
||||
@ -221,7 +221,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
# @unittest.skip("skip")
|
||||
def test_pgpro434_3(self):
|
||||
"""Check pg_stop_backup_timeout, needed backup_timeout"""
|
||||
"""
|
||||
Check pg_stop_backup_timeout, needed backup_timeout
|
||||
Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
node = self.make_simple_node(
|
||||
@ -236,40 +239,32 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
self.add_instance(backup_dir, 'node', node)
|
||||
self.set_archiving(backup_dir, 'node', node)
|
||||
|
||||
archive_script_path = os.path.join(backup_dir, 'archive_script.sh')
|
||||
with open(archive_script_path, 'w+') as f:
|
||||
f.write(
|
||||
archive_script.format(
|
||||
backup_dir=backup_dir, node_name='node', count_limit=2))
|
||||
|
||||
st = os.stat(archive_script_path)
|
||||
os.chmod(archive_script_path, st.st_mode | 0o111)
|
||||
node.append_conf(
|
||||
'postgresql.auto.conf', "archive_command = '{0} %p %f'".format(
|
||||
archive_script_path))
|
||||
|
||||
node.slow_start()
|
||||
|
||||
try:
|
||||
self.backup_node(
|
||||
gdb = self.backup_node(
|
||||
backup_dir, 'node', node,
|
||||
options=[
|
||||
"--archive-timeout=60",
|
||||
"--stream"]
|
||||
)
|
||||
# we should die here because exception is what we expect to happen
|
||||
self.assertEqual(
|
||||
1, 0,
|
||||
"Expecting Error because pg_stop_backup failed to answer.\n "
|
||||
"Output: {0} \n CMD: {1}".format(
|
||||
repr(self.output), self.cmd))
|
||||
"--stream",
|
||||
"--log-level-file=info"],
|
||||
gdb=True)
|
||||
|
||||
except ProbackupException as e:
|
||||
self.assertTrue(
|
||||
"ERROR: pg_stop_backup doesn't answer" in e.message and
|
||||
"cancel it" in e.message,
|
||||
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
|
||||
repr(e.message), self.cmd))
|
||||
gdb.set_breakpoint('pg_stop_backup')
|
||||
gdb.run_until_break()
|
||||
|
||||
node.append_conf(
|
||||
'postgresql.auto.conf', "archive_command = 'exit 1'")
|
||||
node.reload()
|
||||
|
||||
gdb.continue_execution_until_exit()
|
||||
|
||||
log_file = os.path.join(backup_dir, 'log/pg_probackup.log')
|
||||
with open(log_file, 'r') as f:
|
||||
log_content = f.read()
|
||||
self.assertNotIn(
|
||||
"ERROR: pg_stop_backup doesn't answer",
|
||||
log_content,
|
||||
"pg_stop_backup timeouted")
|
||||
|
||||
log_file = os.path.join(node.logs_dir, 'postgresql.log')
|
||||
with open(log_file, 'r') as f:
|
||||
@ -331,6 +326,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
wal_src = os.path.join(
|
||||
node.data_dir, 'pg_wal', '000000010000000000000001')
|
||||
|
||||
if self.archive_compress:
|
||||
with open(wal_src, 'rb') as f_in, gzip.open(
|
||||
file, 'wb', compresslevel=1) as f_out:
|
||||
@ -412,7 +408,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
# @unittest.expectedFailure
|
||||
@unittest.skip("skip")
|
||||
# @unittest.skip("skip")
|
||||
def test_replica_archive(self):
|
||||
"""
|
||||
make node without archiving, take stream backup and
|
||||
@ -502,7 +498,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
"postgres",
|
||||
"insert into t_heap as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(512,20680) i")
|
||||
"from generate_series(512,80680) i")
|
||||
|
||||
before = master.safe_psql("postgres", "SELECT * FROM t_heap")
|
||||
|
||||
@ -510,11 +506,13 @@ class ArchiveTest(ProbackupTest, unittest.TestCase):
|
||||
"postgres",
|
||||
"CHECKPOINT")
|
||||
|
||||
self.wait_until_replica_catch_with_master(master, replica)
|
||||
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'replica',
|
||||
replica, backup_type='page',
|
||||
options=[
|
||||
'--archive-timeout=30',
|
||||
'--archive-timeout=60',
|
||||
'--master-db=postgres',
|
||||
'--master-host=localhost',
|
||||
'--master-port={0}'.format(master.port),
|
||||
|
||||
@ -1 +1 @@
|
||||
pg_probackup 2.0.24
|
||||
pg_probackup 2.0.25
|
||||
@ -60,19 +60,6 @@ idx_ptrack = {
|
||||
}
|
||||
}
|
||||
|
||||
archive_script = """
|
||||
#!/bin/bash
|
||||
count=$(ls {backup_dir}/test00* | wc -l)
|
||||
if [ $count -ge {count_limit} ]
|
||||
then
|
||||
exit 1
|
||||
else
|
||||
cp $1 {backup_dir}/wal/{node_name}/$2
|
||||
count=$((count+1))
|
||||
touch {backup_dir}/test00$count
|
||||
exit 0
|
||||
fi
|
||||
"""
|
||||
warning = """
|
||||
Wrong splint in show_pb
|
||||
Original Header:
|
||||
|
||||
@ -3,6 +3,8 @@ import unittest
|
||||
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
|
||||
from datetime import datetime, timedelta
|
||||
import subprocess
|
||||
import gzip
|
||||
import shutil
|
||||
|
||||
module_name = 'page'
|
||||
|
||||
@ -781,7 +783,22 @@ class PageBackupTest(ProbackupTest, unittest.TestCase):
|
||||
wals_dir, f)) and not f.endswith('.backup')]
|
||||
wals = map(str, wals)
|
||||
# file = os.path.join(wals_dir, max(wals))
|
||||
file = os.path.join(wals_dir, '000000010000000000000004')
|
||||
|
||||
if self.archive_compress:
|
||||
original_file = os.path.join(wals_dir, '000000010000000000000004.gz')
|
||||
tmp_file = os.path.join(backup_dir, '000000010000000000000004')
|
||||
|
||||
with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
# drop healthy file
|
||||
os.remove(original_file)
|
||||
file = tmp_file
|
||||
|
||||
else:
|
||||
file = os.path.join(wals_dir, '000000010000000000000004')
|
||||
|
||||
# corrupt file
|
||||
print(file)
|
||||
with open(file, "rb+", 0) as f:
|
||||
f.seek(42)
|
||||
@ -790,7 +807,14 @@ class PageBackupTest(ProbackupTest, unittest.TestCase):
|
||||
f.close
|
||||
|
||||
if self.archive_compress:
|
||||
file = file[:-3]
|
||||
# compress corrupted file and replace with it old file
|
||||
with open(file, 'rb') as f_in, gzip.open(original_file, 'wb', compresslevel=1) as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
file = os.path.join(wals_dir, '000000010000000000000004.gz')
|
||||
|
||||
#if self.archive_compress:
|
||||
# file = file[:-3]
|
||||
|
||||
# Single-thread PAGE backup
|
||||
try:
|
||||
@ -915,9 +939,6 @@ class PageBackupTest(ProbackupTest, unittest.TestCase):
|
||||
print(file_destination)
|
||||
os.rename(file, file_destination)
|
||||
|
||||
if self.archive_compress:
|
||||
file_destination = file_destination[:-3]
|
||||
|
||||
# Single-thread PAGE backup
|
||||
try:
|
||||
self.backup_node(
|
||||
|
||||
109
tests/replica.py
109
tests/replica.py
@ -236,12 +236,6 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
|
||||
pgbench = master.pgbench(
|
||||
options=['-T', '30', '-c', '2', '--no-vacuum'])
|
||||
|
||||
# master.psql(
|
||||
# "postgres",
|
||||
# "insert into t_heap as select i as id, md5(i::text) as text, "
|
||||
# "md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
# "from generate_series(512,25120) i")
|
||||
|
||||
backup_id = self.backup_node(
|
||||
backup_dir, 'replica',
|
||||
replica, backup_type='page',
|
||||
@ -449,106 +443,3 @@ class ReplicaTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
@unittest.skip("skip")
|
||||
def test_make_block_from_future(self):
|
||||
"""
|
||||
make archive master, take full backups from master,
|
||||
restore full backup as replica, launch pgbench,
|
||||
"""
|
||||
fname = self.id().split('.')[3]
|
||||
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
|
||||
master = self.make_simple_node(
|
||||
base_dir="{0}/{1}/master".format(module_name, fname),
|
||||
set_replication=True,
|
||||
initdb_params=['--data-checksums'],
|
||||
pg_options={
|
||||
'wal_level': 'replica',
|
||||
'max_wal_senders': '2'}
|
||||
)
|
||||
self.init_pb(backup_dir)
|
||||
self.add_instance(backup_dir, 'master', master)
|
||||
self.set_archiving(backup_dir, 'master', master)
|
||||
# force more frequent wal switch
|
||||
#master.append_conf('postgresql.auto.conf', 'archive_timeout = 10')
|
||||
master.slow_start()
|
||||
|
||||
replica = self.make_simple_node(
|
||||
base_dir="{0}/{1}/replica".format(module_name, fname))
|
||||
replica.cleanup()
|
||||
|
||||
self.backup_node(backup_dir, 'master', master)
|
||||
|
||||
self.restore_node(
|
||||
backup_dir, 'master', replica, options=['-R'])
|
||||
|
||||
# Settings for Replica
|
||||
self.set_archiving(backup_dir, 'replica', replica, replica=True)
|
||||
replica.append_conf(
|
||||
'postgresql.auto.conf', 'port = {0}'.format(replica.port))
|
||||
replica.append_conf(
|
||||
'postgresql.auto.conf', 'hot_standby = on')
|
||||
|
||||
replica.slow_start(replica=True)
|
||||
|
||||
self.add_instance(backup_dir, 'replica', replica)
|
||||
|
||||
replica.safe_psql(
|
||||
'postgres',
|
||||
'checkpoint')
|
||||
|
||||
master.pgbench_init(scale=10)
|
||||
|
||||
self.wait_until_replica_catch_with_master(master, replica)
|
||||
|
||||
|
||||
# print(replica.safe_psql(
|
||||
# 'postgres',
|
||||
# 'select * from pg_catalog.pg_last_xlog_receive_location()'))
|
||||
#
|
||||
# print(replica.safe_psql(
|
||||
# 'postgres',
|
||||
# 'select * from pg_catalog.pg_last_xlog_replay_location()'))
|
||||
#
|
||||
# print(replica.safe_psql(
|
||||
# 'postgres',
|
||||
# 'select * from pg_catalog.pg_control_checkpoint()'))
|
||||
#
|
||||
# replica.safe_psql(
|
||||
# 'postgres',
|
||||
# 'checkpoint')
|
||||
|
||||
pgbench = master.pgbench(options=['-T', '30', '-c', '2', '--no-vacuum'])
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
#self.backup_node(backup_dir, 'replica', replica, options=['--stream'])
|
||||
exit(1)
|
||||
self.backup_node(backup_dir, 'replica', replica)
|
||||
pgbench.wait()
|
||||
|
||||
# pgbench
|
||||
master.safe_psql(
|
||||
"postgres",
|
||||
"create table t_heap as select i as id, md5(i::text) as text, "
|
||||
"md5(repeat(i::text,10))::tsvector as tsvector "
|
||||
"from generate_series(0,256000) i")
|
||||
|
||||
|
||||
master.safe_psql(
|
||||
'postgres',
|
||||
'checkpoint')
|
||||
|
||||
replica.safe_psql(
|
||||
'postgres',
|
||||
'checkpoint')
|
||||
|
||||
replica.safe_psql(
|
||||
'postgres',
|
||||
'select * from pg_')
|
||||
|
||||
self.backup_node(backup_dir, 'replica', replica)
|
||||
exit(1)
|
||||
|
||||
# Clean after yourself
|
||||
self.del_test_dir(module_name, fname)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user