db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge with maste

Browse Source
This commit is contained in:
Konstantin Knizhnik
2018-11-29 19:58:40 +03:00
parent c64fd5a43e c14c5a58c5
commit bb760c38c6
41 changed files with 2958 additions and 2633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data.c
+94 -85
View File
@@ -29,6 +29,9 @@ typedef union DataPage
char data[BLCKSZ];
} DataPage;
static bool
fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed);
#ifdef HAVE_LIBZ
/* Implementation of zlib compression method */
static int32
@@ -484,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
blknum, header.compressed_size, write_buffer_size); */
/* Update CRC */
COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size);
/* write data page */
if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size)
@@ -544,13 +547,13 @@ backup_data_file(backup_files_arg* arguments,
/* reset size summary */
file->read_size = 0;
file->write_size = 0;
INIT_TRADITIONAL_CRC32(file->crc);
INIT_FILE_CRC32(true, file->crc);
/* open backup mode file for read */
in = fopen(file->path, PG_BINARY_R);
if (in == NULL)
{
FIN_TRADITIONAL_CRC32(file->crc);
FIN_FILE_CRC32(true, file->crc);
/*
* If file is not found, this is not en error.
@@ -654,7 +657,7 @@ backup_data_file(backup_files_arg* arguments,
to_path, strerror(errno));
fclose(in);
FIN_TRADITIONAL_CRC32(file->crc);
FIN_FILE_CRC32(true, file->crc);
/*
* If we have pagemap then file in the backup can't be a zero size.
@@ -915,7 +918,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location
struct stat st;
pg_crc32 crc;
INIT_TRADITIONAL_CRC32(crc);
INIT_FILE_CRC32(true, crc);
/* reset size summary */
file->read_size = 0;
@@ -925,7 +928,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location
in = fopen(file->path, PG_BINARY_R);
if (in == NULL)
{
FIN_TRADITIONAL_CRC32(crc);
FIN_FILE_CRC32(true, crc);
file->crc = crc;
/* maybe deleted, it's not error */
@@ -974,7 +977,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location
strerror(errno_tmp));
}
/* update CRC */
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
COMP_FILE_CRC32(true, crc, buf, read_len);
file->read_size += read_len;
}
@@ -1001,14 +1004,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location
strerror(errno_tmp));
}
/* update CRC */
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
COMP_FILE_CRC32(true, crc, buf, read_len);
file->read_size += read_len;
}
file->write_size = (int64) file->read_size;
/* finish CRC calculation and store into pgFile */
FIN_TRADITIONAL_CRC32(crc);
FIN_FILE_CRC32(true, crc);
file->crc = crc;
/* update file permission */
@@ -1082,7 +1085,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
FILE *in = NULL;
FILE *out=NULL;
char buf[XLOG_BLCKSZ];
const char *to_path_p = to_path;
const char *to_path_p;
char to_path_temp[MAXPGPATH];
int errno_temp;
@@ -1090,7 +1093,15 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
char gz_to_path[MAXPGPATH];
gzFile gz_out = NULL;
int gz_tmp = -1;
if (is_compress)
{
snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path);
to_path_p = gz_to_path;
}
else
#endif
to_path_p = to_path;
/* open file for read */
in = fopen(from_path, PG_BINARY_R);
@@ -1098,30 +1109,31 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path,
strerror(errno));
/* Check if possible to skip copying */
if (fileExists(to_path_p, FIO_BACKUP_HOST))
{
if (fileEqualCRC(from_path, to_path_p, is_compress))
return;
/* Do not copy and do not rise error. Just quit as normal. */
else if (!overwrite)
elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p);
}
/* open backup file for write */
#ifdef HAVE_LIBZ
if (is_compress)
{
snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path);
if (!overwrite && fileExists(gz_to_path, FIO_BACKUP_HOST))
elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path);
snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path);
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, &gz_tmp, FIO_BACKUP_HOST);
if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK)
if (gzsetparams(gz_out, instance_config.compress_level, Z_DEFAULT_STRATEGY) != Z_OK)
elog(ERROR, "Cannot set compression level %d to file \"%s\": %s",
compress_level, to_path_temp, get_gz_error(gz_out, errno));
to_path_p = gz_to_path;
instance_config.compress_level, to_path_temp,
get_gz_error(gz_out, errno));
}
else
#endif
{
if (!overwrite && fileExists(to_path, FIO_BACKUP_HOST))
elog(ERROR, "WAL segment \"%s\" already exists.", to_path);
snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path);
out = fio_fopen(to_path_temp, PG_BINARY_W, FIO_BACKUP_HOST);
@@ -1397,70 +1409,13 @@ get_wal_file(const char *from_path, const char *to_path)
* but created in process of backup, such as stream XLOG files,
* PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE.
*/
bool
void
calc_file_checksum(pgFile *file)
{
int in;
ssize_t read_len = 0;
int errno_tmp;
char buf[BLCKSZ];
struct stat st;
pg_crc32 crc;
Assert(S_ISREG(file->mode));
INIT_TRADITIONAL_CRC32(crc);
/* reset size summary */
file->read_size = 0;
file->write_size = 0;
/* open backup mode file for read */
in = fio_open(file->path, O_RDONLY|PG_BINARY, FIO_BACKUP_HOST);
if (in < 0)
{
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
/* maybe deleted, it's not error */
if (errno == ENOENT)
return false;
elog(ERROR, "cannot open source file \"%s\": %s", file->path,
strerror(errno));
}
/* stat source file to change mode of destination file */
if (fio_fstat(in, &st) == -1)
{
fio_close(in);
elog(ERROR, "cannot stat \"%s\": %s", file->path,
strerror(errno));
}
while ((read_len = fio_read(in, buf, sizeof(buf))) > 0)
{
/* update CRC */
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
file->write_size += read_len;
file->read_size += read_len;
}
errno_tmp = errno;
if (read_len < 0)
{
fio_close(in);
elog(ERROR, "cannot read backup mode file \"%s\": %s",
file->path, strerror(errno_tmp));
}
/* finish CRC calculation and store into pgFile */
FIN_TRADITIONAL_CRC32(crc);
file->crc = crc;
fio_close(in);
return true;
file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
}
/*
@@ -1582,14 +1537,14 @@ validate_one_page(Page page, pgFile *file,
/* Valiate pages of datafile in backup one by one */
bool
check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version)
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
uint32 backup_version)
{
size_t read_len = 0;
bool is_valid = true;
FILE *in;
pg_crc32 crc;
bool use_crc32c = (backup_version <= 20021);
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
elog(VERBOSE, "validate relation blocks for file %s", file->name);
@@ -1708,3 +1663,57 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
return is_valid;
}
static bool
fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
{
pg_crc32 crc1;
pg_crc32 crc2;
/* Get checksum of backup file */
#ifdef HAVE_LIBZ
if (path2_is_compressed)
{
char buf [1024];
gzFile gz_in = NULL;
int gz_tmp = -1;
INIT_FILE_CRC32(true, crc2);
gz_in = fio_gzopen(path2, PG_BINARY_R, &gz_tmp, FIO_BACKUP_HOST);
if (gz_in == NULL)
/* File cannot be read */
elog(ERROR,
"Cannot compare WAL file \"%s\" with compressed \"%s\"",
path1, path2);
for (;;)
{
size_t read_len = 0;
read_len = gzread(gz_in, buf, sizeof(buf));
if (read_len != sizeof(buf) && !gzeof(gz_in))
/* An error occurred while reading the file */
elog(ERROR,
"Cannot compare WAL file \"%s\" with compressed \"%s\"",
path1, path2);
COMP_FILE_CRC32(true, crc2, buf, read_len);
if (gzeof(gz_in) || read_len == 0)
break;
}
FIN_FILE_CRC32(true, crc2);
if (fio_gzclose(gz_in, path2, gz_tmp) != 0)
elog(ERROR, "Cannot close compressed WAL file \"%s\": %s",
path2, get_gz_error(gz_in, errno));
}
else
#endif
{
crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
}
/* Get checksum of original file */
crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_LOCAL_HOST);
return EQ_CRC32C(crc1, crc2);
}