mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-01-07 13:40:17 +02:00
[Issue #203] Add new pgBackup attribute 'content-crc', containing crc of backup_content.control file, to detect corruption
This commit is contained in:
parent
e368eb7380
commit
9b0922c6b7
@ -217,8 +217,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
|
||||
|
||||
if (prev_backup)
|
||||
{
|
||||
char prev_backup_filelist_path[MAXPGPATH];
|
||||
|
||||
if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION))
|
||||
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
|
||||
"pg_probackup do not guarantee to be forward compatible. "
|
||||
@ -227,10 +225,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
|
||||
|
||||
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
|
||||
|
||||
join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
|
||||
DATABASE_FILE_LIST);
|
||||
/* Files of previous backup needed by DELTA backup */
|
||||
prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST);
|
||||
prev_backup_filelist = get_backup_filelist(prev_backup, true);
|
||||
|
||||
/* If lsn is not NULL, only pages with higher lsn will be copied. */
|
||||
prev_backup_start_lsn = prev_backup->start_lsn;
|
||||
|
@ -538,17 +538,17 @@ err_proc:
|
||||
* TODO this function only used once. Is it really needed?
|
||||
*/
|
||||
parray *
|
||||
get_backup_filelist(pgBackup *backup)
|
||||
get_backup_filelist(pgBackup *backup, bool strict)
|
||||
{
|
||||
parray *files = NULL;
|
||||
char backup_filelist_path[MAXPGPATH];
|
||||
|
||||
join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
|
||||
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST);
|
||||
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc);
|
||||
|
||||
/* redundant sanity? */
|
||||
if (!files)
|
||||
elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time));
|
||||
elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time));
|
||||
|
||||
return files;
|
||||
}
|
||||
@ -1759,6 +1759,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
|
||||
if (backup->note)
|
||||
fio_fprintf(out, "note = '%s'\n", backup->note);
|
||||
|
||||
if (backup->content_crc != 0)
|
||||
fio_fprintf(out, "content-crc = %u\n", backup->content_crc);
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1813,8 +1816,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
parray *external_list, bool sync)
|
||||
{
|
||||
FILE *out;
|
||||
char path[MAXPGPATH];
|
||||
char path_temp[MAXPGPATH];
|
||||
char control_path[MAXPGPATH];
|
||||
char control_path_temp[MAXPGPATH];
|
||||
size_t i = 0;
|
||||
#define BUFFERSZ 1024*1024
|
||||
char *buf;
|
||||
@ -1822,24 +1825,29 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
int64 uncompressed_size_on_disk = 0;
|
||||
int64 wal_size_on_disk = 0;
|
||||
|
||||
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
|
||||
snprintf(path_temp, sizeof(path_temp), "%s.tmp", path);
|
||||
join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST);
|
||||
snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path);
|
||||
|
||||
out = fopen(path_temp, PG_BINARY_W);
|
||||
out = fopen(control_path_temp, PG_BINARY_W);
|
||||
if (out == NULL)
|
||||
elog(ERROR, "Cannot open file list \"%s\": %s", path_temp,
|
||||
elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp,
|
||||
strerror(errno));
|
||||
|
||||
if (chmod(path_temp, FILE_PERMISSION) == -1)
|
||||
elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp,
|
||||
if (chmod(control_path_temp, FILE_PERMISSION) == -1)
|
||||
elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp,
|
||||
strerror(errno));
|
||||
|
||||
buf = pgut_malloc(BUFFERSZ);
|
||||
setvbuf(out, buf, _IOFBF, BUFFERSZ);
|
||||
|
||||
if (sync)
|
||||
INIT_FILE_CRC32(true, backup->content_crc);
|
||||
|
||||
/* print each file in the list */
|
||||
for (i = 0; i < parray_num(files); i++)
|
||||
{
|
||||
int len = 0;
|
||||
char line[BLCKSZ];
|
||||
pgFile *file = (pgFile *) parray_get(files, i);
|
||||
char *path = file->path; /* for streamed WAL files */
|
||||
|
||||
@ -1873,7 +1881,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
(file->external_dir_num && external_list))
|
||||
path = file->rel_path;
|
||||
|
||||
fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
|
||||
len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
|
||||
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
|
||||
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
|
||||
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
|
||||
@ -1887,32 +1895,40 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
file->dbOid);
|
||||
|
||||
if (file->is_datafile)
|
||||
fprintf(out, ",\"segno\":\"%d\"", file->segno);
|
||||
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
|
||||
|
||||
if (file->linked)
|
||||
fprintf(out, ",\"linked\":\"%s\"", file->linked);
|
||||
len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
|
||||
|
||||
if (file->n_blocks != BLOCKNUM_INVALID)
|
||||
fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks);
|
||||
len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
|
||||
|
||||
fprintf(out, "}\n");
|
||||
sprintf(line+len, "}\n");
|
||||
|
||||
if (sync)
|
||||
COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line));
|
||||
|
||||
fprintf(out, "%s", line);
|
||||
}
|
||||
|
||||
if (sync)
|
||||
FIN_FILE_CRC32(true, backup->content_crc);
|
||||
|
||||
if (fflush(out) != 0)
|
||||
elog(ERROR, "Cannot flush file list \"%s\": %s",
|
||||
path_temp, strerror(errno));
|
||||
control_path_temp, strerror(errno));
|
||||
|
||||
if (sync && fsync(fileno(out)) < 0)
|
||||
elog(ERROR, "Cannot sync file list \"%s\": %s",
|
||||
path_temp, strerror(errno));
|
||||
control_path_temp, strerror(errno));
|
||||
|
||||
if (fclose(out) != 0)
|
||||
elog(ERROR, "Cannot close file list \"%s\": %s",
|
||||
path_temp, strerror(errno));
|
||||
control_path_temp, strerror(errno));
|
||||
|
||||
if (rename(path_temp, path) < 0)
|
||||
if (rename(control_path_temp, control_path) < 0)
|
||||
elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s",
|
||||
path_temp, path, strerror(errno));
|
||||
control_path_temp, control_path, strerror(errno));
|
||||
|
||||
/* use extra variable to avoid reset of previous data_bytes value in case of error */
|
||||
backup->data_bytes = backup_size_on_disk;
|
||||
@ -1975,6 +1991,7 @@ readBackupControlFile(const char *path)
|
||||
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
|
||||
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
|
||||
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
|
||||
{'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT},
|
||||
{0}
|
||||
};
|
||||
|
||||
@ -2242,6 +2259,7 @@ pgBackupInit(pgBackup *backup)
|
||||
backup->root_dir = NULL;
|
||||
backup->files = NULL;
|
||||
backup->note = NULL;
|
||||
backup->content_crc = 0;
|
||||
|
||||
}
|
||||
|
||||
|
36
src/data.c
36
src/data.c
@ -1462,7 +1462,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
||||
*/
|
||||
if (errno == ENOENT)
|
||||
{
|
||||
elog(LOG, "File \"%s\" is not found", file->path);
|
||||
elog(LOG, "File \"%s\" is not found", from_fullpath);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -1472,7 +1472,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
||||
}
|
||||
|
||||
if (file->size % BLCKSZ != 0)
|
||||
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
|
||||
elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size);
|
||||
|
||||
/*
|
||||
* Compute expected number of blocks in the file.
|
||||
@ -1508,8 +1508,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
||||
|
||||
/* Valiate pages of datafile in backup one by one */
|
||||
bool
|
||||
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
uint32 backup_version)
|
||||
check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
|
||||
uint32 checksum_version, uint32 backup_version)
|
||||
{
|
||||
size_t read_len = 0;
|
||||
bool is_valid = true;
|
||||
@ -1517,19 +1517,19 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
pg_crc32 crc;
|
||||
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
|
||||
|
||||
elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path);
|
||||
elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath);
|
||||
|
||||
in = fopen(file->path, PG_BINARY_R);
|
||||
in = fopen(fullpath, PG_BINARY_R);
|
||||
if (in == NULL)
|
||||
{
|
||||
if (errno == ENOENT)
|
||||
{
|
||||
elog(WARNING, "File \"%s\" is not found", file->path);
|
||||
elog(WARNING, "File \"%s\" is not found", fullpath);
|
||||
return false;
|
||||
}
|
||||
|
||||
elog(ERROR, "Cannot open file \"%s\": %s",
|
||||
file->path, strerror(errno));
|
||||
fullpath, strerror(errno));
|
||||
}
|
||||
|
||||
/* calc CRC of backup file */
|
||||
@ -1553,7 +1553,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
|
||||
if (ferror(in))
|
||||
elog(ERROR, "Cannot read header of block %u of \"%s\": %s",
|
||||
blknum, file->path, strerror(errno));
|
||||
blknum, fullpath, strerror(errno));
|
||||
|
||||
if (read_len != sizeof(header))
|
||||
{
|
||||
@ -1562,10 +1562,10 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
else if (read_len != 0 && feof(in))
|
||||
elog(WARNING,
|
||||
"Odd size page found at block %u of \"%s\"",
|
||||
blknum, file->path);
|
||||
blknum, fullpath);
|
||||
else
|
||||
elog(WARNING, "Cannot read header of block %u of \"%s\": %s",
|
||||
blknum, file->path, strerror(errno));
|
||||
blknum, fullpath, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1573,14 +1573,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
|
||||
if (header.block == 0 && header.compressed_size == 0)
|
||||
{
|
||||
elog(VERBOSE, "Skip empty block of \"%s\"", file->path);
|
||||
elog(VERBOSE, "Skip empty block of \"%s\"", fullpath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (header.block < blknum)
|
||||
{
|
||||
elog(WARNING, "Backup is broken at block %u of \"%s\"",
|
||||
blknum, file->path);
|
||||
blknum, fullpath);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1589,7 +1589,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
if (header.compressed_size == PageIsTruncated)
|
||||
{
|
||||
elog(LOG, "Block %u of \"%s\" is truncated",
|
||||
blknum, file->path);
|
||||
blknum, fullpath);
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1600,7 +1600,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
if (read_len != MAXALIGN(header.compressed_size))
|
||||
{
|
||||
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
|
||||
blknum, file->path, read_len, header.compressed_size);
|
||||
blknum, fullpath, read_len, header.compressed_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
&errormsg);
|
||||
if (uncompressed_size < 0 && errormsg != NULL)
|
||||
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
|
||||
blknum, file->path, errormsg);
|
||||
blknum, fullpath, errormsg);
|
||||
|
||||
if (uncompressed_size != BLCKSZ)
|
||||
{
|
||||
@ -1630,7 +1630,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
continue;
|
||||
}
|
||||
elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
|
||||
file->path, uncompressed_size);
|
||||
fullpath, uncompressed_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1676,7 +1676,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
|
||||
if (crc != file->crc)
|
||||
{
|
||||
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
|
||||
file->path, crc, file->crc);
|
||||
fullpath, crc, file->crc);
|
||||
is_valid = false;
|
||||
}
|
||||
|
||||
|
26
src/dir.c
26
src/dir.c
@ -1512,12 +1512,13 @@ bad_format:
|
||||
*/
|
||||
parray *
|
||||
dir_read_file_list(const char *root, const char *external_prefix,
|
||||
const char *file_txt, fio_location location)
|
||||
const char *file_txt, fio_location location, pg_crc32 expected_crc)
|
||||
{
|
||||
FILE *fp;
|
||||
parray *files;
|
||||
char buf[MAXPGPATH * 2];
|
||||
char stdio_buf[STDIO_BUFSIZE];
|
||||
FILE *fp;
|
||||
parray *files;
|
||||
char buf[BLCKSZ];
|
||||
char stdio_buf[STDIO_BUFSIZE];
|
||||
pg_crc32 content_crc = 0;
|
||||
|
||||
fp = fio_open_stream(file_txt, location);
|
||||
if (fp == NULL)
|
||||
@ -1529,6 +1530,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
||||
|
||||
files = parray_new();
|
||||
|
||||
INIT_FILE_CRC32(true, content_crc);
|
||||
|
||||
while (fgets(buf, lengthof(buf), fp))
|
||||
{
|
||||
char path[MAXPGPATH];
|
||||
@ -1546,6 +1549,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
||||
dbOid; /* used for partial restore */
|
||||
pgFile *file;
|
||||
|
||||
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
|
||||
|
||||
get_control_value(buf, "path", path, NULL, true);
|
||||
get_control_value(buf, "size", NULL, &write_size, true);
|
||||
get_control_value(buf, "mode", NULL, &mode, true);
|
||||
@ -1598,10 +1603,21 @@ dir_read_file_list(const char *root, const char *external_prefix,
|
||||
parray_append(files, file);
|
||||
}
|
||||
|
||||
FIN_FILE_CRC32(true, content_crc);
|
||||
|
||||
if (ferror(fp))
|
||||
elog(ERROR, "Failed to read from file: \"%s\"", file_txt);
|
||||
|
||||
fio_close_stream(fp);
|
||||
|
||||
if (expected_crc != 0 &&
|
||||
expected_crc != content_crc)
|
||||
{
|
||||
elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u",
|
||||
file_txt, content_crc, expected_crc);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
|
@ -565,13 +565,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
|
||||
*/
|
||||
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||
{
|
||||
char control_file[MAXPGPATH];
|
||||
|
||||
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
|
||||
|
||||
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
|
||||
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
||||
|
||||
backup->files = get_backup_filelist(backup, true);
|
||||
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
|
||||
|
||||
/* Set MERGING status for every member of the chain */
|
||||
|
@ -394,6 +394,8 @@ struct pgBackup
|
||||
parray *files; /* list of files belonging to this backup
|
||||
* must be populated explicitly */
|
||||
char *note;
|
||||
|
||||
pg_crc32 content_crc;
|
||||
};
|
||||
|
||||
/* Recovery target for restore and validate subcommands */
|
||||
@ -708,7 +710,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
|
||||
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
|
||||
PartialRestoreType partial_restore_type);
|
||||
|
||||
extern parray *get_backup_filelist(pgBackup *backup);
|
||||
extern parray *get_backup_filelist(pgBackup *backup, bool strict);
|
||||
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
|
||||
|
||||
/* in merge.c */
|
||||
@ -867,7 +869,7 @@ extern void db_map_entry_free(void *map);
|
||||
extern void print_file_list(FILE *out, const parray *files, const char *root,
|
||||
const char *external_prefix, parray *external_list);
|
||||
extern parray *dir_read_file_list(const char *root, const char *external_prefix,
|
||||
const char *file_txt, fio_location location);
|
||||
const char *file_txt, fio_location location, pg_crc32 expected_crc);
|
||||
extern parray *make_external_directory_list(const char *colon_separated_dirs,
|
||||
bool remap);
|
||||
extern void free_dir_list(parray *list);
|
||||
@ -933,7 +935,7 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
|
||||
extern bool create_empty_file(fio_location from_location, const char *to_root,
|
||||
fio_location to_location, pgFile *file);
|
||||
|
||||
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
|
||||
extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
|
||||
uint32 checksum_version, uint32 backup_version);
|
||||
/* parsexlog.c */
|
||||
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,
|
||||
|
@ -494,7 +494,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
const char *pgdata_path, bool no_sync)
|
||||
{
|
||||
int i;
|
||||
char control_file[MAXPGPATH];
|
||||
char timestamp[100];
|
||||
parray *dest_files = NULL;
|
||||
parray *external_dirs = NULL;
|
||||
@ -515,8 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
|
||||
elog(INFO, "Restoring the database from backup at %s", timestamp);
|
||||
|
||||
join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST);
|
||||
dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
||||
dest_files = get_backup_filelist(dest_backup, true);
|
||||
|
||||
/* Lock backup chain and make sanity checks */
|
||||
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||
@ -550,10 +548,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
|
||||
/* populate backup filelist */
|
||||
if (backup->start_time != dest_backup->start_time)
|
||||
{
|
||||
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
|
||||
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
|
||||
}
|
||||
backup->files = get_backup_filelist(backup, true);
|
||||
else
|
||||
backup->files = dest_files;
|
||||
|
||||
@ -1496,7 +1491,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
|
||||
char database_map_path[MAXPGPATH];
|
||||
parray *files = NULL;
|
||||
|
||||
files = get_backup_filelist(backup);
|
||||
files = get_backup_filelist(backup, true);
|
||||
|
||||
/* look for 'database_map' file in backup_content.control */
|
||||
for (i = 0; i < parray_num(files); i++)
|
||||
|
@ -430,11 +430,16 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup)
|
||||
json_add_value(buf, "status", status2str(backup->status), json_level,
|
||||
true);
|
||||
|
||||
if (backup->note){
|
||||
if (backup->note)
|
||||
json_add_value(buf, "note", backup->note,
|
||||
json_level, true);
|
||||
|
||||
if (backup->content_crc != 0)
|
||||
{
|
||||
json_add_key(buf, "content-crc", json_level);
|
||||
appendPQExpBuffer(buf, "%u", backup->content_crc);
|
||||
}
|
||||
|
||||
json_add(buf, JT_END_OBJECT, &json_level);
|
||||
}
|
||||
|
||||
|
@ -31,6 +31,7 @@ typedef struct
|
||||
uint32 backup_version;
|
||||
BackupMode backup_mode;
|
||||
parray *dbOid_exclude_list;
|
||||
const char *external_prefix;
|
||||
|
||||
/*
|
||||
* Return value from the thread.
|
||||
@ -48,8 +49,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
||||
{
|
||||
char base_path[MAXPGPATH];
|
||||
char external_prefix[MAXPGPATH];
|
||||
char path[MAXPGPATH];
|
||||
parray *files;
|
||||
parray *files = NULL;
|
||||
bool corrupted = false;
|
||||
bool validation_isok = true;
|
||||
/* arrays with meta info for multi threaded validate */
|
||||
@ -110,8 +110,15 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
||||
|
||||
join_path_components(base_path, backup->root_dir, DATABASE_DIR);
|
||||
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
|
||||
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
|
||||
files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST);
|
||||
files = get_backup_filelist(backup, false);
|
||||
|
||||
if (!files)
|
||||
{
|
||||
elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time));
|
||||
backup->status = BACKUP_STATUS_CORRUPT;
|
||||
write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true);
|
||||
return;
|
||||
}
|
||||
|
||||
// if (params && params->partial_db_list)
|
||||
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
|
||||
@ -142,6 +149,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
||||
arg->stop_lsn = backup->stop_lsn;
|
||||
arg->checksum_version = backup->checksum_version;
|
||||
arg->backup_version = parse_program_version(backup->program_version);
|
||||
arg->external_prefix = external_prefix;
|
||||
// arg->dbOid_exclude_list = dbOid_exclude_list;
|
||||
/* By default there are some error */
|
||||
threads_args[i].ret = 1;
|
||||
@ -223,6 +231,7 @@ pgBackupValidateFiles(void *arg)
|
||||
{
|
||||
struct stat st;
|
||||
pgFile *file = (pgFile *) parray_get(arguments->files, i);
|
||||
char file_fullpath[MAXPGPATH];
|
||||
|
||||
if (interrupted || thread_interrupted)
|
||||
elog(ERROR, "Interrupted during validate");
|
||||
@ -244,14 +253,6 @@ pgBackupValidateFiles(void *arg)
|
||||
// continue;
|
||||
//}
|
||||
|
||||
/*
|
||||
* Currently we don't compute checksums for
|
||||
* cfs_compressed data files, so skip them.
|
||||
* TODO: investigate
|
||||
*/
|
||||
if (file->is_cfs)
|
||||
continue;
|
||||
|
||||
if (!pg_atomic_test_set_flag(&file->lock))
|
||||
continue;
|
||||
|
||||
@ -282,14 +283,24 @@ pgBackupValidateFiles(void *arg)
|
||||
if (file->write_size == 0)
|
||||
continue;
|
||||
|
||||
if (file->external_dir_num)
|
||||
{
|
||||
char temp[MAXPGPATH];
|
||||
|
||||
makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
|
||||
join_path_components(file_fullpath, temp, file->rel_path);
|
||||
}
|
||||
else
|
||||
join_path_components(file_fullpath, arguments->base_path, file->rel_path);
|
||||
|
||||
/* TODO: it is redundant to check file existence using stat */
|
||||
if (stat(file->path, &st) == -1)
|
||||
if (stat(file_fullpath, &st) == -1)
|
||||
{
|
||||
if (errno == ENOENT)
|
||||
elog(WARNING, "Backup file \"%s\" is not found", file->path);
|
||||
elog(WARNING, "Backup file \"%s\" is not found", file_fullpath);
|
||||
else
|
||||
elog(WARNING, "Cannot stat backup file \"%s\": %s",
|
||||
file->path, strerror(errno));
|
||||
file_fullpath, strerror(errno));
|
||||
arguments->corrupted = true;
|
||||
break;
|
||||
}
|
||||
@ -297,7 +308,7 @@ pgBackupValidateFiles(void *arg)
|
||||
if (file->write_size != st.st_size)
|
||||
{
|
||||
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
|
||||
file->path, (unsigned long) st.st_size, file->write_size);
|
||||
file_fullpath, (unsigned long) st.st_size, file->write_size);
|
||||
arguments->corrupted = true;
|
||||
break;
|
||||
}
|
||||
@ -305,8 +316,10 @@ pgBackupValidateFiles(void *arg)
|
||||
/*
|
||||
* If option skip-block-validation is set, compute only file-level CRC for
|
||||
* datafiles, otherwise check them block by block.
|
||||
* Currently we don't compute checksums for
|
||||
* cfs_compressed data files, so skip block validation for them.
|
||||
*/
|
||||
if (!file->is_datafile || skip_block_validation)
|
||||
if (!file->is_datafile || skip_block_validation || file->is_cfs)
|
||||
{
|
||||
/*
|
||||
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
|
||||
@ -326,14 +339,14 @@ pgBackupValidateFiles(void *arg)
|
||||
!file->external_dir_num)
|
||||
crc = get_pgcontrol_checksum(arguments->base_path);
|
||||
else
|
||||
crc = pgFileGetCRC(file->path,
|
||||
crc = pgFileGetCRC(file_fullpath,
|
||||
arguments->backup_version <= 20021 ||
|
||||
arguments->backup_version >= 20025,
|
||||
false);
|
||||
if (crc != file->crc)
|
||||
{
|
||||
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
|
||||
file->path, crc, file->crc);
|
||||
file_fullpath, crc, file->crc);
|
||||
arguments->corrupted = true;
|
||||
}
|
||||
}
|
||||
@ -344,7 +357,7 @@ pgBackupValidateFiles(void *arg)
|
||||
* check page headers, checksums (if enabled)
|
||||
* and compute checksum of the file
|
||||
*/
|
||||
if (!check_file_pages(file, arguments->stop_lsn,
|
||||
if (!check_file_pages(file, file_fullpath, arguments->stop_lsn,
|
||||
arguments->checksum_version,
|
||||
arguments->backup_version))
|
||||
arguments->corrupted = true;
|
||||
|
Loading…
Reference in New Issue
Block a user