db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-12-12 11:45:24 +02:00
Code Issues Releases Activity

Merge commit '60731f3223d19f49a796e42910860418c6691a25' into PGPRO11_DEV

Browse Source
This commit is contained in:
Victor Wagner 2018-10-31 11:38:16 +03:00
commit d50a31bef3
20 changed files with 160 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -33,6 +33,7 @@
/tests/helpers/*pyc
# Extra files
/src/pg_crc.c
/src/datapagemap.c
/src/datapagemap.h
/src/logging.h

33
Makefile
View File

@ -1,16 +1,23 @@
PROGRAM = pg_probackup
OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \
src/pg_probackup.o src/restore.o src/show.o \
src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \
src/xlogreader.o src/streamutil.o src/receivelog.o \
src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \
src/utils/json.o src/utils/thread.o src/merge.o
EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h
# utils
OBJS = src/utils/json.o src/utils/logger.o src/utils/parray.o \
src/utils/pgut.o src/utils/thread.o
INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h
OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \
src/parsexlog.o src/pg_probackup.o src/restore.o src/show.o src/util.o \
src/validate.o
# borrowed files
OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \
src/xlogreader.o
EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \
src/xlogreader.c
INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h
ifdef USE_PGXS
PG_CONFIG = pg_config
@ -46,14 +53,14 @@ all: checksrcdir $(INCLUDES);
$(PROGRAM): $(OBJS)
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@
src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@
src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@
src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c
rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@
src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@
src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h
@ -62,6 +69,8 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@
src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
ifeq (,$(filter 9.5 9.6,$(MAJORVERSION)))

3
gen_probackup_project.pl
View File

@ -7,7 +7,7 @@ my $pgsrc="";
if (@ARGV==1)
{
$pgsrc = shift @ARGV;
if($pgsrc == "--help"){
if($pgsrc eq "--help"){
print STDERR "Usage $0 pg-source-dir \n";
print STDERR "Like this: \n";
print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n";
@ -157,6 +157,7 @@ sub build_pgprobackup
'thread.c'
);
$probackup->AddFile('src/backend/access/transam/xlogreader.c');
$probackup->AddFile('src/backend/utils/hash/pg_crc.c');
$probackup->AddFiles(
'src/bin/pg_basebackup',
'receivelog.c',

1
msvs/template.pg_probackup.vcxproj
View File

@ -165,6 +165,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

1
msvs/template.pg_probackup96.vcxproj
View File

@ -165,6 +165,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
<ClCompile Include="@PGSRC@\bin\pg_rewind\datapagemap.c" />

1
msvs/template.pg_probackup_2.vcxproj
View File

@ -160,6 +160,7 @@
<!-- @PGROOT@\lib;@ADDLIBS@ @PGSRC@ @ADDINCLUDE@ -->
<ItemGroup>
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
<ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

29
src/backup.c
View File

@ -306,7 +306,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
to_path, strerror(errno_tmp));
}
INIT_CRC32C(file->crc);
INIT_TRADITIONAL_CRC32(file->crc);
/* read from stream and write to backup file */
while (1)
@ -332,14 +332,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
{
write_buffer_size = Min(row_length, sizeof(buf));
memcpy(buf, copybuf, write_buffer_size);
COMP_CRC32C(file->crc, buf, write_buffer_size);
COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size);
/* TODO calc checksum*/
if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
{
errno_tmp = errno;
/* oops */
FIN_CRC32C(file->crc);
FIN_TRADITIONAL_CRC32(file->crc);
fclose(out);
PQfinish(conn);
elog(ERROR, "cannot write to \"%s\": %s", to_path,
@ -363,7 +363,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
}
file->write_size = (int64) file->read_size;
FIN_CRC32C(file->crc);
FIN_TRADITIONAL_CRC32(file->crc);
fclose(out);
}
@ -538,8 +538,8 @@ do_backup_instance(void)
}
/*
* It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to
* stort_backup LSN of previous backup
* It`s illegal to take PTRACK backup if LSN from ptrack_control() is not
* equal to stop_lsn of previous backup.
*/
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
@ -1193,7 +1193,8 @@ pg_ptrack_support(void)
/* Now we support only ptrack versions upper than 1.5 */
if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 &&
strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0)
strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0 &&
strcmp(PQgetvalue(res_db, 0, 0), "1.7") != 0)
{
elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0));
PQclear(res_db);
@ -1283,7 +1284,9 @@ pg_ptrack_clear(void)
tblspcOid = atoi(PQgetvalue(res_db, i, 2));
tmp_conn = pgut_connect(dbname);
res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL);
res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()",
0, NULL);
PQclear(res);
sprintf(params[0], "%i", dbOid);
sprintf(params[1], "%i", tblspcOid);
@ -1512,7 +1515,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
if (wait_prev_segment)
elog(LOG, "Looking for segment: %s", wal_segment);
else
elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment);
elog(LOG, "Looking for LSN: %X/%X in segment: %s",
(uint32) (lsn >> 32), (uint32) lsn, wal_segment);
#ifdef HAVE_LIBZ
snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz",
@ -2137,7 +2141,7 @@ backup_files(void *arg)
continue;
}
}
else
else
{
bool skip = false;
@ -2147,7 +2151,7 @@ backup_files(void *arg)
{
calc_file_checksum(file);
/* ...and checksum is the same... */
if (EQ_CRC32C(file->crc, (*prev_file)->crc))
if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc))
skip = true; /* ...skip copying file. */
}
if (skip ||
@ -2648,7 +2652,8 @@ get_last_ptrack_lsn(void)
uint32 lsn_lo;
XLogRecPtr lsn;
res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL);
res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()",
0, NULL);
/* Extract timeline and LSN from results of pg_start_backup() */
XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);

34
src/data.c
View File

@ -416,7 +416,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
blknum, header.compressed_size, write_buffer_size); */
/* Update CRC */
COMP_CRC32C(*crc, write_buffer, write_buffer_size);
COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
/* write data page */
if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
@ -476,13 +476,13 @@ backup_data_file(backup_files_arg* arguments,
/* reset size summary */
file->read_size = 0;
file->write_size = 0;
INIT_CRC32C(file->crc);
INIT_TRADITIONAL_CRC32(file->crc);
/* open backup mode file for read */
in = fopen(file->path, PG_BINARY_R);
if (in == NULL)
{
FIN_CRC32C(file->crc);
FIN_TRADITIONAL_CRC32(file->crc);
/*
* If file is not found, this is not en error.
@ -587,7 +587,7 @@ backup_data_file(backup_files_arg* arguments,
to_path, strerror(errno));
fclose(in);
FIN_CRC32C(file->crc);
FIN_TRADITIONAL_CRC32(file->crc);
/*
* If we have pagemap then file in the backup can't be a zero size.
@ -839,7 +839,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
struct stat st;
pg_crc32 crc;
INIT_CRC32C(crc);
INIT_TRADITIONAL_CRC32(crc);
/* reset size summary */
file->read_size = 0;
@ -849,7 +849,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
in = fopen(file->path, PG_BINARY_R);
if (in == NULL)
{
FIN_CRC32C(crc);
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
/* maybe deleted, it's not error */
@ -898,7 +898,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, buf, read_len);
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
file->read_size += read_len;
}
@ -925,14 +925,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, buf, read_len);
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
file->read_size += read_len;
}
file->write_size = (int64) file->read_size;
/* finish CRC calculation and store into pgFile */
FIN_CRC32C(crc);
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
/* update file permission */
@ -1350,7 +1350,7 @@ calc_file_checksum(pgFile *file)
pg_crc32 crc;
Assert(S_ISREG(file->mode));
INIT_CRC32C(crc);
INIT_TRADITIONAL_CRC32(crc);
/* reset size summary */
file->read_size = 0;
@ -1360,7 +1360,7 @@ calc_file_checksum(pgFile *file)
in = fopen(file->path, PG_BINARY_R);
if (in == NULL)
{
FIN_CRC32C(crc);
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
/* maybe deleted, it's not error */
@ -1387,7 +1387,7 @@ calc_file_checksum(pgFile *file)
break;
/* update CRC */
COMP_CRC32C(crc, buf, read_len);
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
file->write_size += read_len;
file->read_size += read_len;
@ -1402,7 +1402,7 @@ calc_file_checksum(pgFile *file)
}
/* finish CRC calculation and store into pgFile */
FIN_CRC32C(crc);
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
fclose(in);
@ -1501,8 +1501,8 @@ validate_one_page(Page page, pgFile *file,
lsn = PageXLogRecPtrGet(phdr->pd_lsn);
if (lsn > stop_lsn)
elog(WARNING, "File: %s, block %u, checksum is not enabled."
"page is from future: pageLSN %X/%X stopLSN %X/%X",
elog(WARNING, "File: %s, block %u, checksum is not enabled. "
"Page is from future: pageLSN %X/%X stopLSN %X/%X",
file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn,
(uint32) (stop_lsn >> 32), (uint32) stop_lsn);
else
@ -1515,8 +1515,8 @@ validate_one_page(Page page, pgFile *file,
lsn = PageXLogRecPtrGet(phdr->pd_lsn);
if (lsn > stop_lsn)
elog(WARNING, "File: %s, block %u, checksum is correct."
"page is from future: pageLSN %X/%X stopLSN %X/%X",
elog(WARNING, "File: %s, block %u, checksum is correct. "
"Page is from future: pageLSN %X/%X stopLSN %X/%X",
file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn,
(uint32) (stop_lsn >> 32), (uint32) stop_lsn);
else

32
src/dir.c
View File

@ -259,7 +259,7 @@ delete_file:
}
pg_crc32
pgFileGetCRC(const char *file_path)
pgFileGetCRC(const char *file_path, bool use_crc32c)
{
FILE *fp;
pg_crc32 crc = 0;
@ -267,6 +267,28 @@ pgFileGetCRC(const char *file_path)
size_t len;
int errno_tmp;
#define INIT_FILE_CRC32(crc) \
do { \
if (use_crc32c) \
INIT_CRC32C(crc); \
else \
INIT_TRADITIONAL_CRC32(crc); \
} while (0)
#define COMP_FILE_CRC32(crc, data, len) \
do { \
if (use_crc32c) \
COMP_CRC32C((crc), (data), (len)); \
else \
COMP_TRADITIONAL_CRC32(crc, data, len); \
} while (0)
#define FIN_FILE_CRC32(crc) \
do { \
if (use_crc32c) \
FIN_CRC32C(crc); \
else \
FIN_TRADITIONAL_CRC32(crc); \
} while (0)
/* open file in binary read mode */
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
@ -274,20 +296,20 @@ pgFileGetCRC(const char *file_path)
file_path, strerror(errno));
/* calc CRC of backup file */
INIT_CRC32C(crc);
INIT_FILE_CRC32(crc);
while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
COMP_CRC32C(crc, buf, len);
COMP_FILE_CRC32(crc, buf, len);
}
errno_tmp = errno;
if (!feof(fp))
elog(WARNING, "cannot read \"%s\": %s", file_path,
strerror(errno_tmp));
if (len > 0)
COMP_CRC32C(crc, buf, len);
FIN_CRC32C(crc);
COMP_FILE_CRC32(crc, buf, len);
FIN_FILE_CRC32(crc);
fclose(fp);

4
src/merge.c
View File

@ -326,7 +326,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
to_backup->wal_bytes = BYTES_INVALID;
pgBackupWriteFileList(to_backup, files, from_database_path);
write_backup_status(to_backup);
write_backup(to_backup);
/* Cleanup */
pfree(threads_args);
@ -503,7 +503,7 @@ merge_files(void *arg)
* do that.
*/
file->write_size = pgFileSize(to_path_tmp);
file->crc = pgFileGetCRC(to_path_tmp);
file->crc = pgFileGetCRC(to_path_tmp, false);
}
pgFileDelete(file);
}

4
src/parsexlog.c
View File

@ -793,6 +793,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private_data = (XLogPageReadPrivate *) xlogreader->private_data;
targetPageOff = targetPagePtr % private_data->xlog_seg_size;
if (interrupted)
elog(ERROR, "Thread [%d]: Interrupted during WAL reading",
private_data->thread_num);
/*
* See if we need to switch to a new segment because the requested record
* is not in the currently open one.

2
src/pg_probackup.c
View File

@ -17,7 +17,7 @@
#include "utils/thread.h"
const char *PROGRAM_VERSION = "2.0.21";
const char *PROGRAM_VERSION = "2.0.22";
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";

5
src/pg_probackup.h
View File

@ -503,7 +503,7 @@ extern pgFile *pgFileNew(const char *path, bool omit_symlink);
extern pgFile *pgFileInit(const char *path);
extern void pgFileDelete(pgFile *file);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c);
extern int pgFileComparePath(const void *f1, const void *f2);
extern int pgFileComparePathDesc(const void *f1, const void *f2);
extern int pgFileCompareLinked(const void *f1, const void *f2);
@ -564,7 +564,8 @@ extern void remove_not_digit(char *buf, size_t len, const char *str);
extern const char *base36enc(long unsigned int value);
extern char *base36enc_dup(long unsigned int value);
extern long unsigned int base36dec(const char *text);
extern int parse_server_version(char *server_version_str);
extern uint32 parse_server_version(const char *server_version_str);
extern uint32 parse_program_version(const char *program_version);
#ifdef WIN32
#ifdef _DEBUG

36
src/util.c
View File

@ -280,12 +280,14 @@ time2iso(char *buf, size_t len, time_t time)
}
}
/* Parse string representation of the server version */
int
parse_server_version(char *server_version_str)
/*
* Parse string representation of the server version.
*/
uint32
parse_server_version(const char *server_version_str)
{
int nfields;
int result = 0;
uint32 result = 0;
int major_version = 0;
int minor_version = 0;
@ -304,7 +306,31 @@ parse_server_version(char *server_version_str)
result = major_version * 10000;
}
else
elog(ERROR, "Unknown server version format");
elog(ERROR, "Unknown server version format %s", server_version_str);
return result;
}
/*
* Parse string representation of the program version.
*/
uint32
parse_program_version(const char *program_version)
{
int nfields;
int major = 0,
minor = 0,
micro = 0;
uint32 result = 0;
if (program_version == NULL || program_version[0] == '\0')
return 0;
nfields = sscanf(program_version, "%d.%d.%d", &major, &minor, &micro);
if (nfields == 3)
result = major * 10000 + minor * 100 + micro;
else
elog(ERROR, "Unknown program version format %s", program_version);
return result;
}

20
src/validate.c
View File

@ -19,6 +19,8 @@ static void *pgBackupValidateFiles(void *arg);
static void do_validate_instance(void);
static bool corrupted_backup_found = false;
/* Program version of a current backup */
static uint32 validate_backup_version = 0;
typedef struct
{
@ -90,6 +92,11 @@ pgBackupValidate(pgBackup *backup)
pg_atomic_clear_flag(&file->lock);
}
/*
* We use program version to calculate checksum in pgBackupValidateFiles()
*/
validate_backup_version = parse_program_version(backup->program_version);
/* init thread args with own file lists */
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (validate_files_arg *)
@ -205,7 +212,18 @@ pgBackupValidateFiles(void *arg)
break;
}
crc = pgFileGetCRC(file->path);
/*
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
* use CRC-32.
*
* pg_control stores its content and checksum of the content, calculated
* using CRC-32C. If we calculate checksum of the whole pg_control using
* CRC-32C we get same checksum constantly. It might be because of the
* CRC-32C algorithm.
* To avoid this problem we need to use different algorithm, CRC-32 in
* this case.
*/
crc = pgFileGetCRC(file->path, validate_backup_version <= 20021);
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",

3
tests/__init__.py
View File

@ -21,7 +21,6 @@ def load_tests(loader, tests, pattern):
# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup))
# suite.addTests(loader.loadTestsFromModule(logging))
suite.addTests(loader.loadTestsFromModule(compression))
suite.addTests(loader.loadTestsFromModule(compatibility))
suite.addTests(loader.loadTestsFromModule(delete_test))
suite.addTests(loader.loadTestsFromModule(delta))
suite.addTests(loader.loadTestsFromModule(exclude))
@ -62,8 +61,6 @@ def load_tests(loader, tests, pattern):
# logging:
# https://jira.postgrespro.ru/browse/PGPRO-584
# https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md
# ptrack:
# ptrack backup on replica should work correctly
# archive:
# immediate recovery and full recovery
# backward compatibility:

3
tests/backup_test.py
View File

@ -520,3 +520,6 @@ class BackupTest(ProbackupTest, unittest.TestCase):
if self.paranoia:
pgdata_restored = self.pgdata_content(node.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
# Clean after yourself
self.del_test_dir(module_name, fname)

42
tests/delete_test.py
View File

@ -58,47 +58,7 @@ class DeleteTest(ProbackupTest, unittest.TestCase):
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_delete_archive_mix_compress_and_non_compressed_segments(self):
"""delete full backups"""
fname = self.id().split('.')[3]
node = self.make_simple_node(
base_dir="{0}/{1}/node".format(module_name, fname),
initdb_params=['--data-checksums'],
pg_options={'wal_level': 'replica'}
)
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.start()
# full backup
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pgbench.wait()
pgbench.stdout.close()
self.backup_node(backup_dir, 'node', node)
show_backups = self.show_pb(backup_dir, 'node')
id_1 = show_backups[0]['ID']
id_2 = show_backups[1]['ID']
id_3 = show_backups[2]['ID']
self.delete_pb(backup_dir, 'node', id_2)
show_backups = self.show_pb(backup_dir, 'node')
self.assertEqual(show_backups[0]['ID'], id_1)
self.assertEqual(show_backups[1]['ID'], id_3)
# Clean after yourself
self.del_test_dir(module_name, fname)
"""stub"""
# @unittest.skip("skip")
def test_delete_increment_page(self):

2
tests/expected/option_version.out
View File

@ -1 +1 @@
pg_probackup 2.0.21
pg_probackup 2.0.22

17
tests/ptrack_vacuum.py
View File

@ -49,6 +49,10 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
node.safe_psql('postgres', 'vacuum t_heap')
node.safe_psql('postgres', 'checkpoint')
# Make full backup to clean every ptrack
self.backup_node(
backup_dir, 'node', node, options=['-j10', '--stream'])
for i in idx_ptrack:
# get fork size and calculate it in pages
idx_ptrack[i]['old_size'] = self.get_fork_size(node, i)
@ -57,11 +61,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# calculate md5sums for every page of this fork
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
# Make full backup to clean every ptrack
self.backup_node(
backup_dir, 'node', node, options=['-j10', '--stream'])
for i in idx_ptrack:
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size'])
@ -158,11 +157,6 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
'--master-db=postgres',
'--master-port={0}'.format(master.port)])
for i in idx_ptrack:
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size'])
for i in idx_ptrack:
# get fork size and calculate it in pages
idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@ -171,6 +165,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
# calculate md5sums for every page of this fork
idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork(
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size'])
# Delete some rows, vacuum it and make checkpoint
master.safe_psql('postgres', 'delete from t_heap where id%2 = 1')