mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-01-23 11:45:36 +02:00
[Issue #203] Add new pgBackup attribute 'content-crc', containing crc of backup_content.control file, to detect corruption
This commit is contained in:
parent
e368eb7380
commit
9b0922c6b7
@ -217,8 +217,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
|
|||||||
|
|
||||||
if (prev_backup)
|
if (prev_backup)
|
||||||
{
|
{
|
||||||
char prev_backup_filelist_path[MAXPGPATH];
|
|
||||||
|
|
||||||
if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION))
|
if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION))
|
||||||
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
|
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
|
||||||
"pg_probackup do not guarantee to be forward compatible. "
|
"pg_probackup do not guarantee to be forward compatible. "
|
||||||
@ -227,10 +225,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
|
|||||||
|
|
||||||
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
|
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
|
||||||
|
|
||||||
join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
|
|
||||||
DATABASE_FILE_LIST);
|
|
||||||
/* Files of previous backup needed by DELTA backup */
|
/* Files of previous backup needed by DELTA backup */
|
||||||
prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST);
|
prev_backup_filelist = get_backup_filelist(prev_backup, true);
|
||||||
|
|
||||||
/* If lsn is not NULL, only pages with higher lsn will be copied. */
|
/* If lsn is not NULL, only pages with higher lsn will be copied. */
|
||||||
prev_backup_start_lsn = prev_backup->start_lsn;
|
prev_backup_start_lsn = prev_backup->start_lsn;
|
||||||
|
@ -538,17 +538,17 @@ err_proc:
|
|||||||
* TODO this function only used once. Is it really needed?
|
* TODO this function only used once. Is it really needed?
|
||||||
*/
|
*/
|
||||||
parray *
|
parray *
|
||||||
get_backup_filelist(pgBackup *backup)
|
get_backup_filelist(pgBackup *backup, bool strict)
|
||||||
{
|
{
|
||||||
parray *files = NULL;
|
parray *files = NULL;
|
||||||
char backup_filelist_path[MAXPGPATH];
|
char backup_filelist_path[MAXPGPATH];
|
||||||
|
|
||||||
join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
|
join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
|
||||||
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST);
|
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc);
|
||||||
|
|
||||||
/* redundant sanity? */
|
/* redundant sanity? */
|
||||||
if (!files)
|
if (!files)
|
||||||
elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time));
|
elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time));
|
||||||
|
|
||||||
return files;
|
return files;
|
||||||
}
|
}
|
||||||
@ -1759,6 +1759,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
|
|||||||
if (backup->note)
|
if (backup->note)
|
||||||
fio_fprintf(out, "note = '%s'\n", backup->note);
|
fio_fprintf(out, "note = '%s'\n", backup->note);
|
||||||
|
|
||||||
|
if (backup->content_crc != 0)
|
||||||
|
fio_fprintf(out, "content-crc = %u\n", backup->content_crc);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -1813,8 +1816,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
|||||||
parray *external_list, bool sync)
|
parray *external_list, bool sync)
|
||||||
{
|
{
|
||||||
FILE *out;
|
FILE *out;
|
||||||
char path[MAXPGPATH];
|
char control_path[MAXPGPATH];
|
||||||
char path_temp[MAXPGPATH];
|
char control_path_temp[MAXPGPATH];
|
||||||
size_t i = 0;
|
size_t i = 0;
|
||||||
#define BUFFERSZ 1024*1024
|
#define BUFFERSZ 1024*1024
|
||||||
char *buf;
|
char *buf;
|
||||||
@ -1822,24 +1825,29 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
|||||||
int64 uncompressed_size_on_disk = 0;
|
int64 uncompressed_size_on_disk = 0;
|
||||||
int64 wal_size_on_disk = 0;
|
int64 wal_size_on_disk = 0;
|
||||||
|
|
||||||
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
|
join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST);
|
||||||
snprintf(path_temp, sizeof(path_temp), "%s.tmp", path);
|
snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path);
|
||||||
|
|
||||||
out = fopen(path_temp, PG_BINARY_W);
|
out = fopen(control_path_temp, PG_BINARY_W);
|
||||||
if (out == NULL)
|
if (out == NULL)
|
||||||
elog(ERROR, "Cannot open file list \"%s\": %s", path_temp,
|
elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp,
|
||||||
strerror(errno));
|
strerror(errno));
|
||||||
|
|
||||||
if (chmod(path_temp, FILE_PERMISSION) == -1)
|
if (chmod(control_path_temp, FILE_PERMISSION) == -1)
|
||||||
elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp,
|
elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp,
|
||||||
strerror(errno));
|
strerror(errno));
|
||||||
|
|
||||||
buf = pgut_malloc(BUFFERSZ);
|
buf = pgut_malloc(BUFFERSZ);
|
||||||
setvbuf(out, buf, _IOFBF, BUFFERSZ);
|
setvbuf(out, buf, _IOFBF, BUFFERSZ);
|
||||||
|
|
||||||
|
if (sync)
|
||||||
|
INIT_FILE_CRC32(true, backup->content_crc);
|
||||||
|
|
||||||
/* print each file in the list */
|
/* print each file in the list */
|
||||||
for (i = 0; i < parray_num(files); i++)
|
for (i = 0; i < parray_num(files); i++)
|
||||||
{
|
{
|
||||||
|
int len = 0;
|
||||||
|
char line[BLCKSZ];
|
||||||
pgFile *file = (pgFile *) parray_get(files, i);
|
pgFile *file = (pgFile *) parray_get(files, i);
|
||||||
char *path = file->path; /* for streamed WAL files */
|
char *path = file->path; /* for streamed WAL files */
|
||||||
|
|
||||||
@ -1873,7 +1881,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
|||||||
(file->external_dir_num && external_list))
|
(file->external_dir_num && external_list))
|
||||||
path = file->rel_path;
|
path = file->rel_path;
|
||||||
|
|
||||||
fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
|
len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
|
||||||
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
|
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
|
||||||
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
|
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
|
||||||
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
|
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
|
||||||
@ -1887,32 +1895,40 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
|||||||
file->dbOid);
|
file->dbOid);
|
||||||
|
|
||||||
if (file->is_datafile)
|
if (file->is_datafile)
|
||||||
fprintf(out, ",\"segno\":\"%d\"", file->segno);
|
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
|
||||||
|
|
||||||
if (file->linked)
|
if (file->linked)
|
||||||
fprintf(out, ",\"linked\":\"%s\"", file->linked);
|
len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
|
||||||
|
|
||||||
if (file->n_blocks != BLOCKNUM_INVALID)
|
if (file->n_blocks != BLOCKNUM_INVALID)
|
||||||
fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks);
|
len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
|
||||||
|
|
||||||
fprintf(out, "}\n");
|
sprintf(line+len, "}\n");
|
||||||
|
|
||||||
|
if (sync)
|
||||||
|
COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line));
|
||||||
|
|
||||||
|
fprintf(out, "%s", line);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (sync)
|
||||||
|
FIN_FILE_CRC32(true, backup->content_crc);
|
||||||
|
|
||||||
if (fflush(out) != 0)
|
if (fflush(out) != 0)
|
||||||
elog(ERROR, "Cannot flush file list \"%s\": %s",
|
elog(ERROR, "Cannot flush file list \"%s\": %s",
|
||||||
path_temp, strerror(errno));
|
control_path_temp, strerror(errno));
|
||||||
|
|
||||||
if (sync && fsync(fileno(out)) < 0)
|
if (sync && fsync(fileno(out)) < 0)
|
||||||
elog(ERROR, "Cannot sync file list \"%s\": %s",
|
elog(ERROR, "Cannot sync file list \"%s\": %s",
|
||||||
path_temp, strerror(errno));
|
control_path_temp, strerror(errno));
|
||||||
|
|
||||||
if (fclose(out) != 0)
|
if (fclose(out) != 0)
|
||||||
elog(ERROR, "Cannot close file list \"%s\": %s",
|
elog(ERROR, "Cannot close file list \"%s\": %s",
|
||||||
path_temp, strerror(errno));
|
control_path_temp, strerror(errno));
|
||||||
|
|
||||||
if (rename(path_temp, path) < 0)
|
if (rename(control_path_temp, control_path) < 0)
|
||||||
elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s",
|
elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s",
|
||||||
path_temp, path, strerror(errno));
|
control_path_temp, control_path, strerror(errno));
|
||||||
|
|
||||||
/* use extra variable to avoid reset of previous data_bytes value in case of error */
|
/* use extra variable to avoid reset of previous data_bytes value in case of error */
|
||||||
backup->data_bytes = backup_size_on_disk;
|
backup->data_bytes = backup_size_on_disk;
|
||||||
@ -1975,6 +1991,7 @@ readBackupControlFile(const char *path)
|
|||||||
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
|
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
|
||||||
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
|
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
|
||||||
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
|
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
|
||||||
|
{'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT},
|
||||||
{0}
|
{0}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -2242,6 +2259,7 @@ pgBackupInit(pgBackup *backup)
|
|||||||
backup->root_dir = NULL;
|
backup->root_dir = NULL;
|
||||||
backup->files = NULL;
|
backup->files = NULL;
|
||||||
backup->note = NULL;
|
backup->note = NULL;
|
||||||
|
backup->content_crc = 0;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
36
src/data.c
36
src/data.c
@ -1462,7 +1462,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
|||||||
*/
|
*/
|
||||||
if (errno == ENOENT)
|
if (errno == ENOENT)
|
||||||
{
|
{
|
||||||
elog(LOG, "File \"%s\" is not found", file->path);
|
elog(LOG, "File \"%s\" is not found", from_fullpath);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1472,7 +1472,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (file->size % BLCKSZ != 0)
|
if (file->size % BLCKSZ != 0)
|
||||||
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
|
elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Compute expected number of blocks in the file.
|
* Compute expected number of blocks in the file.
|
||||||
@ -1508,8 +1508,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
|||||||
|
|
||||||
/* Valiate pages of datafile in backup one by one */
|
/* Valiate pages of datafile in backup one by one */
|
||||||
bool
|
bool
|
||||||
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
|
||||||
uint32 backup_version)
|
uint32 checksum_version, uint32 backup_version)
|
||||||
{
|
{
|
||||||
size_t read_len = 0;
|
size_t read_len = 0;
|
||||||
bool is_valid = true;
|
bool is_valid = true;
|
||||||
@ -1517,19 +1517,19 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
pg_crc32 crc;
|
pg_crc32 crc;
|
||||||
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
|
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
|
||||||
|
|
||||||
elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path);
|
elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath);
|
||||||
|
|
||||||
in = fopen(file->path, PG_BINARY_R);
|
in = fopen(fullpath, PG_BINARY_R);
|
||||||
if (in == NULL)
|
if (in == NULL)
|
||||||
{
|
{
|
||||||
if (errno == ENOENT)
|
if (errno == ENOENT)
|
||||||
{
|
{
|
||||||
elog(WARNING, "File \"%s\" is not found", file->path);
|
elog(WARNING, "File \"%s\" is not found", fullpath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
elog(ERROR, "Cannot open file \"%s\": %s",
|
elog(ERROR, "Cannot open file \"%s\": %s",
|
||||||
file->path, strerror(errno));
|
fullpath, strerror(errno));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* calc CRC of backup file */
|
/* calc CRC of backup file */
|
||||||
@ -1553,7 +1553,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
|
|
||||||
if (ferror(in))
|
if (ferror(in))
|
||||||
elog(ERROR, "Cannot read header of block %u of \"%s\": %s",
|
elog(ERROR, "Cannot read header of block %u of \"%s\": %s",
|
||||||
blknum, file->path, strerror(errno));
|
blknum, fullpath, strerror(errno));
|
||||||
|
|
||||||
if (read_len != sizeof(header))
|
if (read_len != sizeof(header))
|
||||||
{
|
{
|
||||||
@ -1562,10 +1562,10 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
else if (read_len != 0 && feof(in))
|
else if (read_len != 0 && feof(in))
|
||||||
elog(WARNING,
|
elog(WARNING,
|
||||||
"Odd size page found at block %u of \"%s\"",
|
"Odd size page found at block %u of \"%s\"",
|
||||||
blknum, file->path);
|
blknum, fullpath);
|
||||||
else
|
else
|
||||||
elog(WARNING, "Cannot read header of block %u of \"%s\": %s",
|
elog(WARNING, "Cannot read header of block %u of \"%s\": %s",
|
||||||
blknum, file->path, strerror(errno));
|
blknum, fullpath, strerror(errno));
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1573,14 +1573,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
|
|
||||||
if (header.block == 0 && header.compressed_size == 0)
|
if (header.block == 0 && header.compressed_size == 0)
|
||||||
{
|
{
|
||||||
elog(VERBOSE, "Skip empty block of \"%s\"", file->path);
|
elog(VERBOSE, "Skip empty block of \"%s\"", fullpath);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (header.block < blknum)
|
if (header.block < blknum)
|
||||||
{
|
{
|
||||||
elog(WARNING, "Backup is broken at block %u of \"%s\"",
|
elog(WARNING, "Backup is broken at block %u of \"%s\"",
|
||||||
blknum, file->path);
|
blknum, fullpath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1589,7 +1589,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
if (header.compressed_size == PageIsTruncated)
|
if (header.compressed_size == PageIsTruncated)
|
||||||
{
|
{
|
||||||
elog(LOG, "Block %u of \"%s\" is truncated",
|
elog(LOG, "Block %u of \"%s\" is truncated",
|
||||||
blknum, file->path);
|
blknum, fullpath);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1600,7 +1600,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
if (read_len != MAXALIGN(header.compressed_size))
|
if (read_len != MAXALIGN(header.compressed_size))
|
||||||
{
|
{
|
||||||
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
|
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
|
||||||
blknum, file->path, read_len, header.compressed_size);
|
blknum, fullpath, read_len, header.compressed_size);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
&errormsg);
|
&errormsg);
|
||||||
if (uncompressed_size < 0 && errormsg != NULL)
|
if (uncompressed_size < 0 && errormsg != NULL)
|
||||||
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
|
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
|
||||||
blknum, file->path, errormsg);
|
blknum, fullpath, errormsg);
|
||||||
|
|
||||||
if (uncompressed_size != BLCKSZ)
|
if (uncompressed_size != BLCKSZ)
|
||||||
{
|
{
|
||||||
@ -1630,7 +1630,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
|
elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
|
||||||
file->path, uncompressed_size);
|
fullpath, uncompressed_size);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1676,7 +1676,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
|||||||
if (crc != file->crc)
|
if (crc != file->crc)
|
||||||
{
|
{
|
||||||
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
|
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
|
||||||
file->path, crc, file->crc);
|
fullpath, crc, file->crc);
|
||||||
is_valid = false;
|
is_valid = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
20
src/dir.c
20
src/dir.c
@ -1512,12 +1512,13 @@ bad_format:
|
|||||||
*/
|
*/
|
||||||
parray *
|
parray *
|
||||||
dir_read_file_list(const char *root, const char *external_prefix,
|
dir_read_file_list(const char *root, const char *external_prefix,
|
||||||
const char *file_txt, fio_location location)
|
const char *file_txt, fio_location location, pg_crc32 expected_crc)
|
||||||
{
|
{
|
||||||
FILE *fp;
|
FILE *fp;
|
||||||
parray *files;
|
parray *files;
|
||||||
char buf[MAXPGPATH * 2];
|
char buf[BLCKSZ];
|
||||||
char stdio_buf[STDIO_BUFSIZE];
|
char stdio_buf[STDIO_BUFSIZE];
|
||||||
|
pg_crc32 content_crc = 0;
|
||||||
|
|
||||||
fp = fio_open_stream(file_txt, location);
|
fp = fio_open_stream(file_txt, location);
|
||||||
if (fp == NULL)
|
if (fp == NULL)
|
||||||
@ -1529,6 +1530,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
|||||||
|
|
||||||
files = parray_new();
|
files = parray_new();
|
||||||
|
|
||||||
|
INIT_FILE_CRC32(true, content_crc);
|
||||||
|
|
||||||
while (fgets(buf, lengthof(buf), fp))
|
while (fgets(buf, lengthof(buf), fp))
|
||||||
{
|
{
|
||||||
char path[MAXPGPATH];
|
char path[MAXPGPATH];
|
||||||
@ -1546,6 +1549,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
|||||||
dbOid; /* used for partial restore */
|
dbOid; /* used for partial restore */
|
||||||
pgFile *file;
|
pgFile *file;
|
||||||
|
|
||||||
|
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
|
||||||
|
|
||||||
get_control_value(buf, "path", path, NULL, true);
|
get_control_value(buf, "path", path, NULL, true);
|
||||||
get_control_value(buf, "size", NULL, &write_size, true);
|
get_control_value(buf, "size", NULL, &write_size, true);
|
||||||
get_control_value(buf, "mode", NULL, &mode, true);
|
get_control_value(buf, "mode", NULL, &mode, true);
|
||||||
@ -1598,10 +1603,21 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
|||||||
parray_append(files, file);
|
parray_append(files, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FIN_FILE_CRC32(true, content_crc);
|
||||||
|
|
||||||
if (ferror(fp))
|
if (ferror(fp))
|
||||||
elog(ERROR, "Failed to read from file: \"%s\"", file_txt);
|
elog(ERROR, "Failed to read from file: \"%s\"", file_txt);
|
||||||
|
|
||||||
fio_close_stream(fp);
|
fio_close_stream(fp);
|
||||||
|
|
||||||
|
if (expected_crc != 0 &&
|
||||||
|
expected_crc != content_crc)
|
||||||
|
{
|
||||||
|
elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u",
|
||||||
|
file_txt, content_crc, expected_crc);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
return files;
|
return files;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -565,13 +565,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
|
|||||||
*/
|
*/
|
||||||
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||||
{
|
{
|
||||||
char control_file[MAXPGPATH];
|
|
||||||
|
|
||||||
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
|
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
|
||||||
|
|
||||||
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
|
backup->files = get_backup_filelist(backup, true);
|
||||||
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
|
||||||
|
|
||||||
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
|
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
|
||||||
|
|
||||||
/* Set MERGING status for every member of the chain */
|
/* Set MERGING status for every member of the chain */
|
||||||
|
@ -394,6 +394,8 @@ struct pgBackup
|
|||||||
parray *files; /* list of files belonging to this backup
|
parray *files; /* list of files belonging to this backup
|
||||||
* must be populated explicitly */
|
* must be populated explicitly */
|
||||||
char *note;
|
char *note;
|
||||||
|
|
||||||
|
pg_crc32 content_crc;
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Recovery target for restore and validate subcommands */
|
/* Recovery target for restore and validate subcommands */
|
||||||
@ -708,7 +710,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
|
|||||||
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
|
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
|
||||||
PartialRestoreType partial_restore_type);
|
PartialRestoreType partial_restore_type);
|
||||||
|
|
||||||
extern parray *get_backup_filelist(pgBackup *backup);
|
extern parray *get_backup_filelist(pgBackup *backup, bool strict);
|
||||||
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
|
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
|
||||||
|
|
||||||
/* in merge.c */
|
/* in merge.c */
|
||||||
@ -867,7 +869,7 @@ extern void db_map_entry_free(void *map);
|
|||||||
extern void print_file_list(FILE *out, const parray *files, const char *root,
|
extern void print_file_list(FILE *out, const parray *files, const char *root,
|
||||||
const char *external_prefix, parray *external_list);
|
const char *external_prefix, parray *external_list);
|
||||||
extern parray *dir_read_file_list(const char *root, const char *external_prefix,
|
extern parray *dir_read_file_list(const char *root, const char *external_prefix,
|
||||||
const char *file_txt, fio_location location);
|
const char *file_txt, fio_location location, pg_crc32 expected_crc);
|
||||||
extern parray *make_external_directory_list(const char *colon_separated_dirs,
|
extern parray *make_external_directory_list(const char *colon_separated_dirs,
|
||||||
bool remap);
|
bool remap);
|
||||||
extern void free_dir_list(parray *list);
|
extern void free_dir_list(parray *list);
|
||||||
@ -933,7 +935,7 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
|
|||||||
extern bool create_empty_file(fio_location from_location, const char *to_root,
|
extern bool create_empty_file(fio_location from_location, const char *to_root,
|
||||||
fio_location to_location, pgFile *file);
|
fio_location to_location, pgFile *file);
|
||||||
|
|
||||||
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
|
extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
|
||||||
uint32 checksum_version, uint32 backup_version);
|
uint32 checksum_version, uint32 backup_version);
|
||||||
/* parsexlog.c */
|
/* parsexlog.c */
|
||||||
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,
|
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,
|
||||||
|
@ -494,7 +494,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
|||||||
const char *pgdata_path, bool no_sync)
|
const char *pgdata_path, bool no_sync)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
char control_file[MAXPGPATH];
|
|
||||||
char timestamp[100];
|
char timestamp[100];
|
||||||
parray *dest_files = NULL;
|
parray *dest_files = NULL;
|
||||||
parray *external_dirs = NULL;
|
parray *external_dirs = NULL;
|
||||||
@ -515,8 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
|||||||
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
|
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
|
||||||
elog(INFO, "Restoring the database from backup at %s", timestamp);
|
elog(INFO, "Restoring the database from backup at %s", timestamp);
|
||||||
|
|
||||||
join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST);
|
dest_files = get_backup_filelist(dest_backup, true);
|
||||||
dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
|
||||||
|
|
||||||
/* Lock backup chain and make sanity checks */
|
/* Lock backup chain and make sanity checks */
|
||||||
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||||
@ -550,10 +548,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
|||||||
|
|
||||||
/* populate backup filelist */
|
/* populate backup filelist */
|
||||||
if (backup->start_time != dest_backup->start_time)
|
if (backup->start_time != dest_backup->start_time)
|
||||||
{
|
backup->files = get_backup_filelist(backup, true);
|
||||||
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
|
|
||||||
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
backup->files = dest_files;
|
backup->files = dest_files;
|
||||||
|
|
||||||
@ -1496,7 +1491,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
|
|||||||
char database_map_path[MAXPGPATH];
|
char database_map_path[MAXPGPATH];
|
||||||
parray *files = NULL;
|
parray *files = NULL;
|
||||||
|
|
||||||
files = get_backup_filelist(backup);
|
files = get_backup_filelist(backup, true);
|
||||||
|
|
||||||
/* look for 'database_map' file in backup_content.control */
|
/* look for 'database_map' file in backup_content.control */
|
||||||
for (i = 0; i < parray_num(files); i++)
|
for (i = 0; i < parray_num(files); i++)
|
||||||
|
@ -430,11 +430,16 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup)
|
|||||||
json_add_value(buf, "status", status2str(backup->status), json_level,
|
json_add_value(buf, "status", status2str(backup->status), json_level,
|
||||||
true);
|
true);
|
||||||
|
|
||||||
if (backup->note){
|
if (backup->note)
|
||||||
json_add_value(buf, "note", backup->note,
|
json_add_value(buf, "note", backup->note,
|
||||||
json_level, true);
|
json_level, true);
|
||||||
|
|
||||||
|
if (backup->content_crc != 0)
|
||||||
|
{
|
||||||
|
json_add_key(buf, "content-crc", json_level);
|
||||||
|
appendPQExpBuffer(buf, "%u", backup->content_crc);
|
||||||
}
|
}
|
||||||
|
|
||||||
json_add(buf, JT_END_OBJECT, &json_level);
|
json_add(buf, JT_END_OBJECT, &json_level);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@ typedef struct
|
|||||||
uint32 backup_version;
|
uint32 backup_version;
|
||||||
BackupMode backup_mode;
|
BackupMode backup_mode;
|
||||||
parray *dbOid_exclude_list;
|
parray *dbOid_exclude_list;
|
||||||
|
const char *external_prefix;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Return value from the thread.
|
* Return value from the thread.
|
||||||
@ -48,8 +49,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
|||||||
{
|
{
|
||||||
char base_path[MAXPGPATH];
|
char base_path[MAXPGPATH];
|
||||||
char external_prefix[MAXPGPATH];
|
char external_prefix[MAXPGPATH];
|
||||||
char path[MAXPGPATH];
|
parray *files = NULL;
|
||||||
parray *files;
|
|
||||||
bool corrupted = false;
|
bool corrupted = false;
|
||||||
bool validation_isok = true;
|
bool validation_isok = true;
|
||||||
/* arrays with meta info for multi threaded validate */
|
/* arrays with meta info for multi threaded validate */
|
||||||
@ -110,8 +110,15 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
|||||||
|
|
||||||
join_path_components(base_path, backup->root_dir, DATABASE_DIR);
|
join_path_components(base_path, backup->root_dir, DATABASE_DIR);
|
||||||
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
|
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
|
||||||
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
|
files = get_backup_filelist(backup, false);
|
||||||
files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST);
|
|
||||||
|
if (!files)
|
||||||
|
{
|
||||||
|
elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time));
|
||||||
|
backup->status = BACKUP_STATUS_CORRUPT;
|
||||||
|
write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// if (params && params->partial_db_list)
|
// if (params && params->partial_db_list)
|
||||||
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
|
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
|
||||||
@ -142,6 +149,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
|||||||
arg->stop_lsn = backup->stop_lsn;
|
arg->stop_lsn = backup->stop_lsn;
|
||||||
arg->checksum_version = backup->checksum_version;
|
arg->checksum_version = backup->checksum_version;
|
||||||
arg->backup_version = parse_program_version(backup->program_version);
|
arg->backup_version = parse_program_version(backup->program_version);
|
||||||
|
arg->external_prefix = external_prefix;
|
||||||
// arg->dbOid_exclude_list = dbOid_exclude_list;
|
// arg->dbOid_exclude_list = dbOid_exclude_list;
|
||||||
/* By default there are some error */
|
/* By default there are some error */
|
||||||
threads_args[i].ret = 1;
|
threads_args[i].ret = 1;
|
||||||
@ -223,6 +231,7 @@ pgBackupValidateFiles(void *arg)
|
|||||||
{
|
{
|
||||||
struct stat st;
|
struct stat st;
|
||||||
pgFile *file = (pgFile *) parray_get(arguments->files, i);
|
pgFile *file = (pgFile *) parray_get(arguments->files, i);
|
||||||
|
char file_fullpath[MAXPGPATH];
|
||||||
|
|
||||||
if (interrupted || thread_interrupted)
|
if (interrupted || thread_interrupted)
|
||||||
elog(ERROR, "Interrupted during validate");
|
elog(ERROR, "Interrupted during validate");
|
||||||
@ -244,14 +253,6 @@ pgBackupValidateFiles(void *arg)
|
|||||||
// continue;
|
// continue;
|
||||||
//}
|
//}
|
||||||
|
|
||||||
/*
|
|
||||||
* Currently we don't compute checksums for
|
|
||||||
* cfs_compressed data files, so skip them.
|
|
||||||
* TODO: investigate
|
|
||||||
*/
|
|
||||||
if (file->is_cfs)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (!pg_atomic_test_set_flag(&file->lock))
|
if (!pg_atomic_test_set_flag(&file->lock))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
@ -282,14 +283,24 @@ pgBackupValidateFiles(void *arg)
|
|||||||
if (file->write_size == 0)
|
if (file->write_size == 0)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
|
if (file->external_dir_num)
|
||||||
|
{
|
||||||
|
char temp[MAXPGPATH];
|
||||||
|
|
||||||
|
makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
|
||||||
|
join_path_components(file_fullpath, temp, file->rel_path);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
join_path_components(file_fullpath, arguments->base_path, file->rel_path);
|
||||||
|
|
||||||
/* TODO: it is redundant to check file existence using stat */
|
/* TODO: it is redundant to check file existence using stat */
|
||||||
if (stat(file->path, &st) == -1)
|
if (stat(file_fullpath, &st) == -1)
|
||||||
{
|
{
|
||||||
if (errno == ENOENT)
|
if (errno == ENOENT)
|
||||||
elog(WARNING, "Backup file \"%s\" is not found", file->path);
|
elog(WARNING, "Backup file \"%s\" is not found", file_fullpath);
|
||||||
else
|
else
|
||||||
elog(WARNING, "Cannot stat backup file \"%s\": %s",
|
elog(WARNING, "Cannot stat backup file \"%s\": %s",
|
||||||
file->path, strerror(errno));
|
file_fullpath, strerror(errno));
|
||||||
arguments->corrupted = true;
|
arguments->corrupted = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -297,7 +308,7 @@ pgBackupValidateFiles(void *arg)
|
|||||||
if (file->write_size != st.st_size)
|
if (file->write_size != st.st_size)
|
||||||
{
|
{
|
||||||
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
|
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
|
||||||
file->path, (unsigned long) st.st_size, file->write_size);
|
file_fullpath, (unsigned long) st.st_size, file->write_size);
|
||||||
arguments->corrupted = true;
|
arguments->corrupted = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -305,8 +316,10 @@ pgBackupValidateFiles(void *arg)
|
|||||||
/*
|
/*
|
||||||
* If option skip-block-validation is set, compute only file-level CRC for
|
* If option skip-block-validation is set, compute only file-level CRC for
|
||||||
* datafiles, otherwise check them block by block.
|
* datafiles, otherwise check them block by block.
|
||||||
|
* Currently we don't compute checksums for
|
||||||
|
* cfs_compressed data files, so skip block validation for them.
|
||||||
*/
|
*/
|
||||||
if (!file->is_datafile || skip_block_validation)
|
if (!file->is_datafile || skip_block_validation || file->is_cfs)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
|
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
|
||||||
@ -326,14 +339,14 @@ pgBackupValidateFiles(void *arg)
|
|||||||
!file->external_dir_num)
|
!file->external_dir_num)
|
||||||
crc = get_pgcontrol_checksum(arguments->base_path);
|
crc = get_pgcontrol_checksum(arguments->base_path);
|
||||||
else
|
else
|
||||||
crc = pgFileGetCRC(file->path,
|
crc = pgFileGetCRC(file_fullpath,
|
||||||
arguments->backup_version <= 20021 ||
|
arguments->backup_version <= 20021 ||
|
||||||
arguments->backup_version >= 20025,
|
arguments->backup_version >= 20025,
|
||||||
false);
|
false);
|
||||||
if (crc != file->crc)
|
if (crc != file->crc)
|
||||||
{
|
{
|
||||||
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
|
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
|
||||||
file->path, crc, file->crc);
|
file_fullpath, crc, file->crc);
|
||||||
arguments->corrupted = true;
|
arguments->corrupted = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -344,7 +357,7 @@ pgBackupValidateFiles(void *arg)
|
|||||||
* check page headers, checksums (if enabled)
|
* check page headers, checksums (if enabled)
|
||||||
* and compute checksum of the file
|
* and compute checksum of the file
|
||||||
*/
|
*/
|
||||||
if (!check_file_pages(file, arguments->stop_lsn,
|
if (!check_file_pages(file, file_fullpath, arguments->stop_lsn,
|
||||||
arguments->checksum_version,
|
arguments->checksum_version,
|
||||||
arguments->backup_version))
|
arguments->backup_version))
|
||||||
arguments->corrupted = true;
|
arguments->corrupted = true;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user