db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-19 20:00:16 +02:00
Code Issues Releases Activity

Merge branch 'REL_2_5' into REL_2_5-PBCKP-304

Browse Source
This commit is contained in:
Yura Sokolov 2022-11-15 17:35:22 +03:00 committed by Ivan Lazarev
commit e2b4549887
16 changed files with 938 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packaging/Makefile.test
View File

@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2
@echo Suse: done
build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13
@echo Rhel 15.1: done
@echo Suse 15.1: done
build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14
@echo Rhel 15.1: done
@echo Suse 15.2: done
define test_suse
docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \

6
packaging/test/scripts/rpm.sh
View File

@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then
dnf -qy module disable postgresql
fi
# PGDG doesn't support install of PG-9.6 from repo package anymore
if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then
# ugly hack: use repo settings from PG10
sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo
fi
yum install -y postgresql${PG_TOG}-server.x86_64
export PGDATA=/var/lib/pgsql/${PG_VERSION}/data

8
src/archive.c
View File

@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
#ifdef HAVE_LIBZ
/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
if (IsXLogFileName(filename))
rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg);
rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg);
if (rc == FILE_MISSING)
#endif
/* ... failing that, use uncompressed */
rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg);
/* When not in prefetch mode, try to use partial file */
if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
#ifdef HAVE_LIBZ
/* '.gz.partial' goes first ... */
snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg);
rc = fio_send_file_gz(from_partial, out, &errmsg);
if (rc == FILE_MISSING)
#endif
{
/* ... failing that, use '.partial' */
snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
rc = fio_send_file(from_partial, out, false, NULL, &errmsg);
}
if (rc == SEND_OK)

17
src/backup.c
View File

@ -946,10 +946,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo)
*/
#ifdef PGPRO_VERSION
if (!res)
{
/* It seems we connected to PostgreSQL (not Postgres Pro) */
elog(ERROR, "%s was built with Postgres Pro %s %s, "
"but connection is made with PostgreSQL %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
if(strcmp(PGPRO_EDITION, "1C") != 0)
{
elog(ERROR, "%s was built with Postgres Pro %s %s, "
"but connection is made with PostgreSQL %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
}
/* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C
* Check the major version
*/
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0)
elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s",
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
}
else
{
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 &&

30
src/catalog.c
View File

@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
char linked[MAXPGPATH];
char compress_alg_string[MAXPGPATH];
int64 write_size,
uncompressed_size,
mode, /* bit length of mode_t depends on platforms */
is_datafile,
is_cfs,
@ -1132,8 +1133,30 @@ get_backup_filelist(pgBackup *backup, bool strict)
if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
file->hdr_size = (int) hdr_size;
if (file->external_dir_num == 0)
if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
file->uncompressed_size = uncompressed_size;
else
file->uncompressed_size = write_size;
if (!file->is_datafile || file->is_cfs)
file->size = file->uncompressed_size;
if (file->external_dir_num == 0 && S_ISREG(file->mode))
{
bool is_datafile = file->is_datafile;
set_forkname(file);
if (is_datafile != file->is_datafile)
{
if (is_datafile)
elog(WARNING, "File '%s' was stored as datafile, but looks like it is not",
file->rel_path);
else
elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is",
file->rel_path);
/* Lets fail in tests */
Assert(file->is_datafile == file->is_datafile);
file->is_datafile = is_datafile;
}
}
parray_append(files, file);
}
@ -2561,6 +2584,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
file->external_dir_num,
file->dbOid);
if (file->uncompressed_size != 0 &&
file->uncompressed_size != file->write_size)
len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
file->uncompressed_size);
if (file->is_datafile)
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);

134
src/data.c
View File

@ -799,13 +799,17 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
* and its mtime is less than parent backup start time ... */
if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
(prev_file && file->exists_in_prev &&
file->size == prev_file->size &&
file->mtime <= parent_backup_time))
{
/*
* file could be deleted under our feets.
* But then backup_non_data_file_internal will handle it safely
*/
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
if (file->forkName != cfm)
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
else
file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true);
/* ...and checksum is the same... */
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@ -1330,7 +1334,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
if (already_exists)
{
/* compare checksums of already existing file and backup file */
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
pg_crc32 file_crc;
if (tmp_file->forkName == cfm &&
tmp_file->uncompressed_size > tmp_file->write_size)
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false);
else
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
if (file_crc == tmp_file->crc)
{
@ -1387,10 +1396,12 @@ backup_non_data_file_internal(const char *from_fullpath,
const char *to_fullpath, pgFile *file,
bool missing_ok)
{
FILE *in = NULL;
FILE *out = NULL;
ssize_t read_len = 0;
char *buf = NULL;
char *errmsg = NULL;
int rc;
bool cut_zero_tail;
cut_zero_tail = file->forkName == cfm;
INIT_FILE_CRC32(true, file->crc);
@ -1412,107 +1423,44 @@ backup_non_data_file_internal(const char *from_fullpath,
/* backup remote file */
if (fio_is_remote(FIO_DB_HOST))
{
char *errmsg = NULL;
int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
/* handle errors */
if (rc == FILE_MISSING)
{
/* maybe deleted, it's not error in case of backup */
if (missing_ok)
{
elog(LOG, "File \"%s\" is not found", from_fullpath);
file->write_size = FILE_NOT_FOUND;
goto cleanup;
}
else
elog(ERROR, "File \"%s\" is not found", from_fullpath);
}
else if (rc == WRITE_FAILED)
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
else if (rc != SEND_OK)
{
if (errmsg)
elog(ERROR, "%s", errmsg);
else
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
}
pg_free(errmsg);
}
/* backup local file */
rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
else
rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
/* handle errors */
if (rc == FILE_MISSING)
{
/* open source file for read */
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
/* maybe deleted, it's not error in case of backup */
if (missing_ok)
{
/* maybe deleted, it's not error in case of backup */
if (errno == ENOENT)
{
if (missing_ok)
{
elog(LOG, "File \"%s\" is not found", from_fullpath);
file->write_size = FILE_NOT_FOUND;
goto cleanup;
}
else
elog(ERROR, "File \"%s\" is not found", from_fullpath);
}
elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
strerror(errno));
}
/* disable stdio buffering for local input/output files to avoid triple buffering */
setvbuf(in, NULL, _IONBF, BUFSIZ);
setvbuf(out, NULL, _IONBF, BUFSIZ);
/* allocate 64kB buffer */
buf = pgut_malloc(CHUNK_SIZE);
/* copy content and calc CRC */
for (;;)
{
read_len = fread(buf, 1, CHUNK_SIZE, in);
if (ferror(in))
elog(ERROR, "Cannot read from file \"%s\": %s",
from_fullpath, strerror(errno));
if (read_len > 0)
{
if (fwrite(buf, 1, read_len, out) != read_len)
elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
strerror(errno));
/* update CRC */
COMP_FILE_CRC32(true, file->crc, buf, read_len);
file->read_size += read_len;
}
if (feof(in))
break;
elog(LOG, "File \"%s\" is not found", from_fullpath);
file->write_size = FILE_NOT_FOUND;
goto cleanup;
}
else
elog(ERROR, "File \"%s\" is not found", from_fullpath);
}
else if (rc == WRITE_FAILED)
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
else if (rc != SEND_OK)
{
if (errmsg)
elog(ERROR, "%s", errmsg);
else
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
}
file->write_size = (int64) file->read_size;
if (file->write_size > 0)
file->uncompressed_size = file->write_size;
file->uncompressed_size = file->read_size;
cleanup:
if (errmsg != NULL)
pg_free(errmsg);
/* finish CRC calculation and store into pgFile */
FIN_FILE_CRC32(true, file->crc);
if (in && fclose(in))
elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
if (out && fclose(out))
elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
pg_free(buf);
}
/*

227
src/dir.c
View File

@ -262,137 +262,6 @@ delete_file:
}
}
/*
* Read the local file to compute its CRC.
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
*/
pg_crc32
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
pg_crc32 crc = 0;
char *buf;
size_t len = 0;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
/* disable stdio buffering */
setvbuf(fp, NULL, _IONBF, BUFSIZ);
buf = pgut_malloc(STDIO_BUFSIZE);
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = fread(buf, 1, STDIO_BUFSIZE, fp);
if (ferror(fp))
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
if (feof(fp))
break;
}
FIN_FILE_CRC32(use_crc32c, crc);
fclose(fp);
pg_free(buf);
return crc;
}
/*
* Read the local file to compute its CRC.
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
*/
pg_crc32
pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
{
gzFile fp;
pg_crc32 crc = 0;
int len = 0;
int err;
char *buf;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = gzopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
buf = pgut_malloc(STDIO_BUFSIZE);
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = gzread(fp, buf, STDIO_BUFSIZE);
if (len <= 0)
{
/* we either run into eof or error */
if (gzeof(fp))
break;
else
{
const char *err_str = NULL;
err_str = gzerror(fp, &err);
elog(ERROR, "Cannot read from compressed file %s", err_str);
}
}
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
}
FIN_FILE_CRC32(use_crc32c, crc);
gzclose(fp);
pg_free(buf);
return crc;
}
void
pgFileFree(void *file)
{
@ -762,20 +631,6 @@ dir_check_file(pgFile *file, bool backup_logs)
if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */
return CHECK_FALSE;
else if (file->forkName != none)
return CHECK_TRUE;
/* Set is_datafile flag */
{
char suffix[MAXFNAMELEN];
/* check if file is datafile */
sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
&(file->segno), suffix);
Assert(sscanf_res > 0); /* since first char is digit */
if (sscanf_res == 1 || sscanf_res == 2)
file->is_datafile = true;
}
}
}
@ -1812,7 +1667,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
FIO_BACKUP_HOST);
file->crc = pgFileGetCRC(database_map_path, true, false);
file->write_size = file->size;
file->uncompressed_size = file->read_size;
file->uncompressed_size = file->size;
parray_append(backup_files_list, file);
}
@ -1920,34 +1775,74 @@ pfilearray_clear_locks(parray *file_list)
}
}
static inline bool
is_forkname(char *name, size_t *pos, const char *forkname)
{
size_t fnlen = strlen(forkname);
if (strncmp(name + *pos, forkname, fnlen) != 0)
return false;
*pos += fnlen;
return true;
}
#define OIDCHARS 10
/* Set forkName if possible */
void
bool
set_forkname(pgFile *file)
{
int name_len = strlen(file->name);
size_t i = 0;
uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */
/* Auxiliary fork of the relfile */
if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0)
/* pretend it is not relation file */
file->relOid = 0;
file->forkName = none;
file->is_datafile = false;
for (i = 0; isdigit(file->name[i]); i++)
{
if (i == 0 && file->name[i] == '0')
return false;
oid = oid * 10 + file->name[i] - '0';
}
if (i == 0 || i > OIDCHARS || oid > UINT32_MAX)
return false;
/* usual fork name */
/* /^\d+_(vm|fsm|init|ptrack)$/ */
if (is_forkname(file->name, &i, "_vm"))
file->forkName = vm;
else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0)
else if (is_forkname(file->name, &i, "_fsm"))
file->forkName = fsm;
else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0)
file->forkName = cfm;
else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0)
else if (is_forkname(file->name, &i, "_init"))
file->forkName = init;
else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0)
else if (is_forkname(file->name, &i, "_ptrack"))
file->forkName = ptrack;
// extract relOid for certain forks
/* segment number */
/* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */
if (file->name[i] == '.' && isdigit(file->name[i+1]))
{
for (i++; isdigit(file->name[i]); i++)
;
}
if ((file->forkName == vm ||
file->forkName == fsm ||
file->forkName == init ||
file->forkName == cfm) &&
(sscanf(file->name, "%u*", &(file->relOid)) != 1))
file->relOid = 0;
/* CFS "fork name" */
if (file->forkName == none &&
is_forkname(file->name, &i, ".cfm"))
{
/* /^\d+(\.\d+)?.cfm$/ */
file->forkName = cfm;
}
/* If there are excess characters, it is not relation file */
if (file->name[i] != 0)
{
file->forkName = none;
return false;
}
file->relOid = oid;
file->is_datafile = file->forkName == none;
return true;
}

2
src/merge.c
View File

@ -1078,7 +1078,7 @@ merge_files(void *arg)
tmp_file->hdr_crc = file->hdr_crc;
}
else
tmp_file->uncompressed_size = tmp_file->write_size;
tmp_file->uncompressed_size = tmp_file->uncompressed_size;
/* Copy header metadata from old map into a new one */
tmp_file->n_headers = file->n_headers;

11
src/pg_probackup.h
View File

@ -345,7 +345,7 @@ typedef enum ShowFormat
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.5.8"
#define PROGRAM_VERSION "2.5.9"
/* update when remote agent API or behaviour changes */
#define AGENT_PROTOCOL_VERSION 20509
@ -1082,6 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok);
extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok);
extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok);
extern int pgFileMapComparePath(const void *f1, const void *f2);
@ -1097,7 +1098,7 @@ extern int pgCompareString(const void *str1, const void *str2);
extern int pgPrefixCompareString(const void *str1, const void *str2);
extern int pgCompareOid(const void *f1, const void *f2);
extern void pfilearray_clear_locks(parray *file_list);
extern void set_forkname(pgFile *file);
extern bool set_forkname(pgFile *file);
/* in data.c */
extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,
@ -1244,9 +1245,11 @@ extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pg
XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version,
bool use_pagemap, BlockNumber *err_blknum, char **errormsg);
/* return codes for fio_send_pages */
extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg);
extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg);
extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail,
pgFile *file, char **errormsg);
extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail,
pgFile *file, char **errormsg);
extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num);

561
src/utils/file.c
View File

@ -18,6 +18,10 @@ static __thread int fio_stdin = 0;
static __thread int fio_stderr = 0;
static char *async_errormsg = NULL;
#define PAGE_ZEROSEARCH_COARSE_GRANULARITY 4096
#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64
static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0};
fio_location MyLocation;
typedef struct
@ -1362,14 +1366,18 @@ fio_sync(char const* path, fio_location location)
enum {
GET_CRC32_DECOMPRESS = 1,
GET_CRC32_MISSING_OK = 2
GET_CRC32_MISSING_OK = 2,
GET_CRC32_TRUNCATED = 4
};
/* Get crc32 of file */
pg_crc32
fio_get_crc32(const char *file_path, fio_location location,
bool decompress, bool missing_ok)
static pg_crc32
fio_get_crc32_ex(const char *file_path, fio_location location,
bool decompress, bool missing_ok, bool truncated)
{
if (decompress && truncated)
elog(ERROR, "Could not calculate CRC for compressed truncated file");
if (fio_is_remote(location))
{
fio_header hdr;
@ -1384,6 +1392,8 @@ fio_get_crc32(const char *file_path, fio_location location,
hdr.arg = GET_CRC32_DECOMPRESS;
if (missing_ok)
hdr.arg |= GET_CRC32_MISSING_OK;
if (truncated)
hdr.arg |= GET_CRC32_TRUNCATED;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len);
@ -1395,11 +1405,27 @@ fio_get_crc32(const char *file_path, fio_location location,
{
if (decompress)
return pgFileGetCRCgz(file_path, true, missing_ok);
else if (truncated)
return pgFileGetCRCTruncated(file_path, true, missing_ok);
else
return pgFileGetCRC(file_path, true, missing_ok);
}
}
pg_crc32
fio_get_crc32(const char *file_path, fio_location location,
bool decompress, bool missing_ok)
{
return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false);
}
pg_crc32
fio_get_crc32_truncated(const char *file_path, fio_location location,
bool missing_ok)
{
return fio_get_crc32_ex(file_path, location, false, missing_ok, true);
}
/* Remove file */
int
fio_unlink(char const* path, fio_location location)
@ -2460,7 +2486,7 @@ cleanup:
* REMOTE_ERROR (-6)
*/
int
fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg)
fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg)
{
fio_header hdr;
int exit_code = SEND_OK;
@ -2609,6 +2635,105 @@ cleanup:
return exit_code;
}
typedef struct send_file_state {
bool calc_crc;
uint32_t crc;
int64_t read_size;
int64_t write_size;
} send_file_state;
/* find page border of all-zero tail */
static size_t
find_zero_tail(char *buf, size_t len)
{
size_t i, l;
size_t granul = sizeof(zerobuf);
if (len == 0)
return 0;
/* fast check for last bytes */
l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY);
i = len - l;
if (memcmp(buf + i, zerobuf, l) != 0)
return len;
/* coarse search for zero tail */
i = (len-1) & ~(granul-1);
l = len - i;
for (;;)
{
if (memcmp(buf+i, zerobuf, l) != 0)
{
i += l;
break;
}
if (i == 0)
break;
i -= granul;
l = granul;
}
len = i;
/* search zero tail with finer granularity */
for (granul = sizeof(zerobuf)/2;
len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY;
granul /= 2)
{
if (granul > l)
continue;
i = (len-1) & ~(granul-1);
l = len - i;
if (memcmp(buf+i, zerobuf, l) == 0)
len = i;
}
return len;
}
static void
fio_send_file_crc(send_file_state* st, char *buf, size_t len)
{
int64_t write_size;
if (!st->calc_crc)
return;
write_size = st->write_size;
while (st->read_size > write_size)
{
size_t crc_len = Min(st->read_size - write_size, sizeof(zerobuf));
COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len);
write_size += crc_len;
}
if (len > 0)
COMP_FILE_CRC32(true, st->crc, buf, len);
}
static bool
fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len)
{
if (len == 0)
return true;
if (st->read_size > st->write_size &&
fseeko(out, st->read_size, SEEK_SET) != 0)
{
return false;
}
if (fwrite(buf, 1, len, out) != len)
{
return false;
}
st->read_size += len;
st->write_size = st->read_size;
return true;
}
/* Receive chunks of data and write them to destination file.
* Return codes:
* SEND_OK (0)
@ -2621,13 +2746,22 @@ cleanup:
* If pgFile is not NULL then we must calculate crc and read_size for it.
*/
int
fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail,
pgFile *file, char **errormsg)
{
fio_header hdr;
int exit_code = SEND_OK;
size_t path_len = strlen(from_fullpath) + 1;
char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */
send_file_state st = {false, 0, 0, 0};
memset(&hdr, 0, sizeof(hdr));
if (file)
{
st.calc_crc = true;
st.crc = file->crc;
}
hdr.cop = FIO_SEND_FILE;
hdr.size = path_len;
@ -2645,6 +2779,37 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
if (hdr.cop == FIO_SEND_FILE_EOF)
{
if (st.write_size < st.read_size)
{
if (!cut_zero_tail)
{
/*
* We still need to calc crc for zero tail.
*/
fio_send_file_crc(&st, NULL, 0);
/*
* Let's write single zero byte to the end of file to restore
* logical size.
* Well, it would be better to use ftruncate here actually,
* but then we need to change interface.
*/
st.read_size -= 1;
buf[0] = 0;
if (!fio_send_file_write(out, &st, buf, 1))
{
exit_code = WRITE_FAILED;
break;
}
}
}
if (file)
{
file->crc = st.crc;
file->read_size = st.read_size;
file->write_size = st.write_size;
}
break;
}
else if (hdr.cop == FIO_ERROR)
@ -2665,17 +2830,23 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size);
/* We have received a chunk of data data, lets write it out */
if (fwrite(buf, 1, hdr.size, out) != hdr.size)
fio_send_file_crc(&st, buf, hdr.size);
if (!fio_send_file_write(out, &st, buf, hdr.size))
{
exit_code = WRITE_FAILED;
break;
}
}
else if (hdr.cop == FIO_PAGE_ZERO)
{
Assert(hdr.size == 0);
Assert(hdr.arg <= CHUNK_SIZE);
if (file)
{
file->read_size += hdr.size;
COMP_FILE_CRC32(true, file->crc, buf, hdr.size);
}
/*
* We have received a chunk of zero data, lets just think we
* wrote it.
*/
st.read_size += hdr.arg;
}
else
{
@ -2691,6 +2862,128 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
return exit_code;
}
int
fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail,
pgFile *file, char **errormsg)
{
FILE* in;
char* buf;
size_t read_len, non_zero_len;
int exit_code = SEND_OK;
send_file_state st = {false, 0, 0, 0};
if (file)
{
st.calc_crc = true;
st.crc = file->crc;
}
/* open source file for read */
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
{
/* maybe deleted, it's not error in case of backup */
if (errno == ENOENT)
return FILE_MISSING;
*errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath,
strerror(errno));
return OPEN_FAILED;
}
/* disable stdio buffering for local input/output files to avoid triple buffering */
setvbuf(in, NULL, _IONBF, BUFSIZ);
setvbuf(out, NULL, _IONBF, BUFSIZ);
/* allocate 64kB buffer */
buf = pgut_malloc(CHUNK_SIZE);
/* copy content and calc CRC */
for (;;)
{
read_len = fread(buf, 1, CHUNK_SIZE, in);
if (ferror(in))
{
*errormsg = psprintf("Cannot read from file \"%s\": %s",
from_fullpath, strerror(errno));
exit_code = READ_FAILED;
goto cleanup;
}
if (read_len > 0)
{
non_zero_len = find_zero_tail(buf, read_len);
/*
* It is dirty trick to silence warnings in CFS GC process:
* backup at least cfs header size bytes.
*/
if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
st.read_size + read_len > 0)
{
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
st.read_size + read_len);
non_zero_len -= st.read_size;
}
if (non_zero_len > 0)
{
fio_send_file_crc(&st, buf, non_zero_len);
if (!fio_send_file_write(out, &st, buf, non_zero_len))
{
exit_code = WRITE_FAILED;
goto cleanup;
}
}
if (non_zero_len < read_len)
{
/* Just pretend we wrote it. */
st.read_size += read_len - non_zero_len;
}
}
if (feof(in))
break;
}
if (st.write_size < st.read_size)
{
if (!cut_zero_tail)
{
/*
* We still need to calc crc for zero tail.
*/
fio_send_file_crc(&st, NULL, 0);
/*
* Let's write single zero byte to the end of file to restore
* logical size.
* Well, it would be better to use ftruncate here actually,
* but then we need to change interface.
*/
st.read_size -= 1;
buf[0] = 0;
if (!fio_send_file_write(out, &st, buf, 1))
{
exit_code = WRITE_FAILED;
goto cleanup;
}
}
}
if (file)
{
file->crc = st.crc;
file->read_size = st.read_size;
file->write_size = st.write_size;
}
cleanup:
free(buf);
fclose(in);
return exit_code;
}
/* Send file content
* On error we return FIO_ERROR message with following codes
* FIO_ERROR:
@ -2709,6 +3002,7 @@ fio_send_file_impl(int out, char const* path)
fio_header hdr;
char *buf = pgut_malloc(CHUNK_SIZE);
size_t read_len = 0;
int64_t read_size = 0;
char *errormsg = NULL;
/* open source file for read */
@ -2751,6 +3045,7 @@ fio_send_file_impl(int out, char const* path)
for (;;)
{
read_len = fread(buf, 1, CHUNK_SIZE, fp);
memset(&hdr, 0, sizeof(hdr));
/* report error */
if (ferror(fp))
@ -2771,10 +3066,36 @@ fio_send_file_impl(int out, char const* path)
if (read_len > 0)
{
/* send chunk */
hdr.cop = FIO_PAGE;
hdr.size = read_len;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(out, buf, read_len), read_len);
int64_t non_zero_len = find_zero_tail(buf, read_len);
/*
* It is dirty trick to silence warnings in CFS GC process:
* backup at least cfs header size bytes.
*/
if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
read_size + read_len > 0)
{
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
read_size + read_len);
non_zero_len -= read_size;
}
if (non_zero_len > 0)
{
hdr.cop = FIO_PAGE;
hdr.size = non_zero_len;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len);
}
if (non_zero_len < read_len)
{
hdr.cop = FIO_PAGE_ZERO;
hdr.size = 0;
hdr.arg = read_len - non_zero_len;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
read_size += read_len;
}
if (feof(fp))
@ -2793,6 +3114,210 @@ cleanup:
return;
}
/*
* Read the local file to compute its CRC.
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
*/
pg_crc32
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
pg_crc32 crc = 0;
char *buf;
size_t len = 0;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
/* disable stdio buffering */
setvbuf(fp, NULL, _IONBF, BUFSIZ);
buf = pgut_malloc(STDIO_BUFSIZE);
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = fread(buf, 1, STDIO_BUFSIZE, fp);
if (ferror(fp))
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
if (feof(fp))
break;
}
FIN_FILE_CRC32(use_crc32c, crc);
fclose(fp);
pg_free(buf);
return crc;
}
/*
* Read the local file to compute CRC for it extened to real_size.
*/
pg_crc32
pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
char *buf;
size_t len = 0;
size_t non_zero_len;
send_file_state st = {true, 0, 0, 0};
INIT_FILE_CRC32(use_crc32c, st.crc);
/* open file in binary read mode */
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, st.crc);
return st.crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
/* disable stdio buffering */
setvbuf(fp, NULL, _IONBF, BUFSIZ);
buf = pgut_malloc(CHUNK_SIZE);
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = fread(buf, 1, STDIO_BUFSIZE, fp);
if (ferror(fp))
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
non_zero_len = find_zero_tail(buf, len);
/* same trick as in fio_send_file */
if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
st.read_size + len > 0)
{
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
st.read_size + len);
non_zero_len -= st.read_size;
}
if (non_zero_len)
{
fio_send_file_crc(&st, buf, non_zero_len);
st.write_size += st.read_size + non_zero_len;
}
st.read_size += len;
if (feof(fp))
break;
}
FIN_FILE_CRC32(use_crc32c, st.crc);
fclose(fp);
pg_free(buf);
return st.crc;
}
/*
* Read the local file to compute its CRC.
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
*/
pg_crc32
pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
{
gzFile fp;
pg_crc32 crc = 0;
int len = 0;
int err;
char *buf;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = gzopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
{
if (missing_ok)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
}
buf = pgut_malloc(STDIO_BUFSIZE);
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = gzread(fp, buf, STDIO_BUFSIZE);
if (len <= 0)
{
/* we either run into eof or error */
if (gzeof(fp))
break;
else
{
const char *err_str = NULL;
err_str = gzerror(fp, &err);
elog(ERROR, "Cannot read from compressed file %s", err_str);
}
}
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
}
FIN_FILE_CRC32(use_crc32c, crc);
gzclose(fp);
pg_free(buf);
return crc;
}
/* Compile the array of files located on remote machine in directory root */
static void
fio_list_dir_internal(parray *files, const char *root, bool exclude,
@ -3399,9 +3924,13 @@ fio_communicate(int in, int out)
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
break;
case FIO_GET_CRC32:
Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 ||
(hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED);
/* calculate crc32 for a file */
if ((hdr.arg & GET_CRC32_DECOMPRESS))
crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
else if ((hdr.arg & GET_CRC32_TRUNCATED))
crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
else
crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));

5
src/utils/file.h
View File

@ -56,7 +56,8 @@ typedef enum
FIO_CHECK_POSTMASTER,
FIO_GET_ASYNC_ERROR,
FIO_WRITE_ASYNC,
FIO_READLINK
FIO_READLINK,
FIO_PAGE_ZERO
} fio_operations;
typedef enum
@ -122,6 +123,8 @@ extern void fio_disconnect(void);
extern int fio_sync(char const* path, fio_location location);
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
bool decompress, bool missing_ok);
extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location,
bool missing_ok);
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

13
tests/backup.py
View File

@ -3251,10 +3251,15 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.assertIn(
'WARNING: backup in progress, stop backup',
log_content)
self.assertIn(
'FROM pg_catalog.pg_backup_stop',
log_content)
if self.get_version(node) < 150000:
self.assertIn(
'FROM pg_catalog.pg_stop_backup',
log_content)
else:
self.assertIn(
'FROM pg_catalog.pg_backup_stop',
log_content)
self.assertIn(
'setting its status to ERROR',

81
tests/cfs_backup.py
View File

@ -169,12 +169,18 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
"ERROR: File pg_compression not found in {0}".format(
os.path.join(self.backup_dir, 'node', backup_id))
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
# check cfm size
cfms = find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm'])
self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
for cfm in cfms:
size = os.stat(cfm).st_size
self.assertLessEqual(size, 4096,
"ERROR: {0} is not truncated (has size {1} > 4096)".format(
cfm, size
))
# @unittest.expectedFailure
# @unittest.skip("skip")
@ -411,6 +417,69 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase):
"ERROR: .cfm files not found in backup dir"
)
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_page_doesnt_store_unchanged_cfm(self):
"""
Case: Test page backup doesn't store cfm file if table were not modified
"""
self.node.safe_psql(
"postgres",
"CREATE TABLE {0} TABLESPACE {1} "
"AS SELECT i AS id, MD5(i::text) AS text, "
"MD5(repeat(i::text,10))::tsvector AS tsvector "
"FROM generate_series(0,256) i".format('t1', tblspace_name)
)
try:
backup_id_full = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='full')
except ProbackupException as e:
self.fail(
"ERROR: Full backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
self.assertTrue(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)],
['.cfm']),
"ERROR: .cfm files not found in backup dir"
)
try:
backup_id = self.backup_node(
self.backup_dir, 'node', self.node, backup_type='page')
except ProbackupException as e:
self.fail(
"ERROR: Incremental backup failed.\n {0} \n {1}".format(
repr(self.cmd),
repr(e.message)
)
)
show_backup = self.show_pb(self.backup_dir, 'node', backup_id)
self.assertEqual(
"OK",
show_backup["status"],
"ERROR: Incremental backup status is not valid. \n "
"Current backup status={0}".format(show_backup["status"])
)
self.assertTrue(
find_by_name(
[self.get_tblspace_path(self.node, tblspace_name)],
['pg_compression']),
"ERROR: File pg_compression not found"
)
self.assertFalse(
find_by_extensions(
[os.path.join(self.backup_dir, 'backups', 'node', backup_id)],
['.cfm']),
"ERROR: .cfm files is found in backup dir"
)
# @unittest.expectedFailure
# @unittest.skip("skip")
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')

127
tests/cfs_catchup.py Normal file
View File

@ -0,0 +1,127 @@
import os
import unittest
import random
import shutil
from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
module_name = 'cfs_catchup'
tblspace_name = 'cfs_tblspace'
class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase):
def setUp(self):
self.fname = self.id().split('.')[3]
@unittest.skipUnless(ProbackupTest.enterprise, 'skip')
def test_full_catchup_with_tablespace(self):
"""
Test tablespace transfers
"""
# preparation
src_pg = self.make_simple_node(
base_dir = os.path.join(module_name, self.fname, 'src'),
set_replication = True
)
src_pg.slow_start()
tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old')
self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True)
src_pg.safe_psql(
"postgres",
"CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer")
src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
src_pg.safe_psql(
"postgres",
"CHECKPOINT")
# do full catchup with tablespace mapping
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new')
self.catchup_node(
backup_mode = 'FULL',
source_pgdata = src_pg.data_dir,
destination_node = dst_pg,
options = [
'-d', 'postgres',
'-p', str(src_pg.port),
'--stream',
'-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path)
]
)
# 1st check: compare data directories
self.compare_pgdata(
self.pgdata_content(src_pg.data_dir),
self.pgdata_content(dst_pg.data_dir)
)
# check cfm size
cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm'])
self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
for cfm in cfms:
size = os.stat(cfm).st_size
self.assertLessEqual(size, 4096,
"ERROR: {0} is not truncated (has size {1} > 4096)".format(
cfm, size
))
# make changes in master tablespace
src_pg.safe_psql(
"postgres",
"UPDATE ultimate_question SET answer = -1")
src_pg.safe_psql(
"postgres",
"CHECKPOINT")
# run&recover catchup'ed instance
dst_options = {}
dst_options['port'] = str(dst_pg.port)
self.set_auto_conf(dst_pg, dst_options)
dst_pg.slow_start()
# 2nd check: run verification query
dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
# and now delta backup
dst_pg.stop()
self.catchup_node(
backup_mode = 'DELTA',
source_pgdata = src_pg.data_dir,
destination_node = dst_pg,
options = [
'-d', 'postgres',
'-p', str(src_pg.port),
'--stream',
'-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path)
]
)
# check cfm size again
cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm'])
self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir")
for cfm in cfms:
size = os.stat(cfm).st_size
self.assertLessEqual(size, 4096,
"ERROR: {0} is not truncated (has size {1} > 4096)".format(
cfm, size
))
# run&recover catchup'ed instance
dst_options = {}
dst_options['port'] = str(dst_pg.port)
self.set_auto_conf(dst_pg, dst_options)
dst_pg.slow_start()
# 3rd check: run verification query
src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy')
# Cleanup
src_pg.stop()
dst_pg.stop()
self.del_test_dir(module_name, self.fname)

2
tests/expected/option_version.out
View File

@ -1 +1 @@
pg_probackup 2.5.8
pg_probackup 2.5.9

14
tests/helpers/ptrack_helpers.py
View File

@ -1751,8 +1751,18 @@ class ProbackupTest(object):
file_relpath = os.path.relpath(file_fullpath, pgdata)
directory_dict['files'][file_relpath] = {'is_datafile': False}
with open(file_fullpath, 'rb') as f:
directory_dict['files'][file_relpath]['md5'] = hashlib.md5(f.read()).hexdigest()
f.close()
content = f.read()
# truncate cfm's content's zero tail
if file_relpath.endswith('.cfm'):
zero64 = b"\x00"*64
l = len(content)
while l > 64:
s = (l - 1) & ~63
if content[s:l] != zero64[:l-s]:
break
l = s
content = content[:l]
directory_dict['files'][file_relpath]['md5'] = hashlib.md5(content).hexdigest()
# directory_dict['files'][file_relpath]['md5'] = hashlib.md5(
# f = open(file_fullpath, 'rb').read()).hexdigest()