1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-23 11:45:36 +02:00

[Issue #203] Add new pgBackup attribute 'content-crc', containing crc of backup_content.control file, to detect corruption

This commit is contained in:
Grigory Smolkin 2020-05-11 15:06:07 +03:00
parent e368eb7380
commit 9b0922c6b7
9 changed files with 127 additions and 86 deletions

View File

@ -217,8 +217,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
if (prev_backup) if (prev_backup)
{ {
char prev_backup_filelist_path[MAXPGPATH];
if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION))
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
"pg_probackup do not guarantee to be forward compatible. " "pg_probackup do not guarantee to be forward compatible. "
@ -227,10 +225,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
DATABASE_FILE_LIST);
/* Files of previous backup needed by DELTA backup */ /* Files of previous backup needed by DELTA backup */
prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST); prev_backup_filelist = get_backup_filelist(prev_backup, true);
/* If lsn is not NULL, only pages with higher lsn will be copied. */ /* If lsn is not NULL, only pages with higher lsn will be copied. */
prev_backup_start_lsn = prev_backup->start_lsn; prev_backup_start_lsn = prev_backup->start_lsn;

View File

@ -538,17 +538,17 @@ err_proc:
* TODO this function only used once. Is it really needed? * TODO this function only used once. Is it really needed?
*/ */
parray * parray *
get_backup_filelist(pgBackup *backup) get_backup_filelist(pgBackup *backup, bool strict)
{ {
parray *files = NULL; parray *files = NULL;
char backup_filelist_path[MAXPGPATH]; char backup_filelist_path[MAXPGPATH];
join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST); files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc);
/* redundant sanity? */ /* redundant sanity? */
if (!files) if (!files)
elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time)); elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time));
return files; return files;
} }
@ -1759,6 +1759,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
if (backup->note) if (backup->note)
fio_fprintf(out, "note = '%s'\n", backup->note); fio_fprintf(out, "note = '%s'\n", backup->note);
if (backup->content_crc != 0)
fio_fprintf(out, "content-crc = %u\n", backup->content_crc);
} }
/* /*
@ -1813,8 +1816,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
parray *external_list, bool sync) parray *external_list, bool sync)
{ {
FILE *out; FILE *out;
char path[MAXPGPATH]; char control_path[MAXPGPATH];
char path_temp[MAXPGPATH]; char control_path_temp[MAXPGPATH];
size_t i = 0; size_t i = 0;
#define BUFFERSZ 1024*1024 #define BUFFERSZ 1024*1024
char *buf; char *buf;
@ -1822,24 +1825,29 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
int64 uncompressed_size_on_disk = 0; int64 uncompressed_size_on_disk = 0;
int64 wal_size_on_disk = 0; int64 wal_size_on_disk = 0;
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST);
snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path);
out = fopen(path_temp, PG_BINARY_W); out = fopen(control_path_temp, PG_BINARY_W);
if (out == NULL) if (out == NULL)
elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp,
strerror(errno)); strerror(errno));
if (chmod(path_temp, FILE_PERMISSION) == -1) if (chmod(control_path_temp, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp,
strerror(errno)); strerror(errno));
buf = pgut_malloc(BUFFERSZ); buf = pgut_malloc(BUFFERSZ);
setvbuf(out, buf, _IOFBF, BUFFERSZ); setvbuf(out, buf, _IOFBF, BUFFERSZ);
if (sync)
INIT_FILE_CRC32(true, backup->content_crc);
/* print each file in the list */ /* print each file in the list */
for (i = 0; i < parray_num(files); i++) for (i = 0; i < parray_num(files); i++)
{ {
int len = 0;
char line[BLCKSZ];
pgFile *file = (pgFile *) parray_get(files, i); pgFile *file = (pgFile *) parray_get(files, i);
char *path = file->path; /* for streamed WAL files */ char *path = file->path; /* for streamed WAL files */
@ -1873,7 +1881,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
(file->external_dir_num && external_list)) (file->external_dir_num && external_list))
path = file->rel_path; path = file->rel_path;
fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
"\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", "
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
@ -1887,32 +1895,40 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
file->dbOid); file->dbOid);
if (file->is_datafile) if (file->is_datafile)
fprintf(out, ",\"segno\":\"%d\"", file->segno); len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
if (file->linked) if (file->linked)
fprintf(out, ",\"linked\":\"%s\"", file->linked); len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
if (file->n_blocks != BLOCKNUM_INVALID) if (file->n_blocks != BLOCKNUM_INVALID)
fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
fprintf(out, "}\n"); sprintf(line+len, "}\n");
if (sync)
COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line));
fprintf(out, "%s", line);
} }
if (sync)
FIN_FILE_CRC32(true, backup->content_crc);
if (fflush(out) != 0) if (fflush(out) != 0)
elog(ERROR, "Cannot flush file list \"%s\": %s", elog(ERROR, "Cannot flush file list \"%s\": %s",
path_temp, strerror(errno)); control_path_temp, strerror(errno));
if (sync && fsync(fileno(out)) < 0) if (sync && fsync(fileno(out)) < 0)
elog(ERROR, "Cannot sync file list \"%s\": %s", elog(ERROR, "Cannot sync file list \"%s\": %s",
path_temp, strerror(errno)); control_path_temp, strerror(errno));
if (fclose(out) != 0) if (fclose(out) != 0)
elog(ERROR, "Cannot close file list \"%s\": %s", elog(ERROR, "Cannot close file list \"%s\": %s",
path_temp, strerror(errno)); control_path_temp, strerror(errno));
if (rename(path_temp, path) < 0) if (rename(control_path_temp, control_path) < 0)
elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s",
path_temp, path, strerror(errno)); control_path_temp, control_path, strerror(errno));
/* use extra variable to avoid reset of previous data_bytes value in case of error */ /* use extra variable to avoid reset of previous data_bytes value in case of error */
backup->data_bytes = backup_size_on_disk; backup->data_bytes = backup_size_on_disk;
@ -1975,6 +1991,7 @@ readBackupControlFile(const char *path)
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT}, {'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
{'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT},
{0} {0}
}; };
@ -2242,6 +2259,7 @@ pgBackupInit(pgBackup *backup)
backup->root_dir = NULL; backup->root_dir = NULL;
backup->files = NULL; backup->files = NULL;
backup->note = NULL; backup->note = NULL;
backup->content_crc = 0;
} }

View File

@ -1462,7 +1462,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
*/ */
if (errno == ENOENT) if (errno == ENOENT)
{ {
elog(LOG, "File \"%s\" is not found", file->path); elog(LOG, "File \"%s\" is not found", from_fullpath);
return true; return true;
} }
@ -1472,7 +1472,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
} }
if (file->size % BLCKSZ != 0) if (file->size % BLCKSZ != 0)
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size); elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size);
/* /*
* Compute expected number of blocks in the file. * Compute expected number of blocks in the file.
@ -1508,8 +1508,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
/* Valiate pages of datafile in backup one by one */ /* Valiate pages of datafile in backup one by one */
bool bool
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 backup_version) uint32 checksum_version, uint32 backup_version)
{ {
size_t read_len = 0; size_t read_len = 0;
bool is_valid = true; bool is_valid = true;
@ -1517,19 +1517,19 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
pg_crc32 crc; pg_crc32 crc;
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path); elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath);
in = fopen(file->path, PG_BINARY_R); in = fopen(fullpath, PG_BINARY_R);
if (in == NULL) if (in == NULL)
{ {
if (errno == ENOENT) if (errno == ENOENT)
{ {
elog(WARNING, "File \"%s\" is not found", file->path); elog(WARNING, "File \"%s\" is not found", fullpath);
return false; return false;
} }
elog(ERROR, "Cannot open file \"%s\": %s", elog(ERROR, "Cannot open file \"%s\": %s",
file->path, strerror(errno)); fullpath, strerror(errno));
} }
/* calc CRC of backup file */ /* calc CRC of backup file */
@ -1553,7 +1553,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (ferror(in)) if (ferror(in))
elog(ERROR, "Cannot read header of block %u of \"%s\": %s", elog(ERROR, "Cannot read header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno)); blknum, fullpath, strerror(errno));
if (read_len != sizeof(header)) if (read_len != sizeof(header))
{ {
@ -1562,10 +1562,10 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
else if (read_len != 0 && feof(in)) else if (read_len != 0 && feof(in))
elog(WARNING, elog(WARNING,
"Odd size page found at block %u of \"%s\"", "Odd size page found at block %u of \"%s\"",
blknum, file->path); blknum, fullpath);
else else
elog(WARNING, "Cannot read header of block %u of \"%s\": %s", elog(WARNING, "Cannot read header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno)); blknum, fullpath, strerror(errno));
return false; return false;
} }
@ -1573,14 +1573,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (header.block == 0 && header.compressed_size == 0) if (header.block == 0 && header.compressed_size == 0)
{ {
elog(VERBOSE, "Skip empty block of \"%s\"", file->path); elog(VERBOSE, "Skip empty block of \"%s\"", fullpath);
continue; continue;
} }
if (header.block < blknum) if (header.block < blknum)
{ {
elog(WARNING, "Backup is broken at block %u of \"%s\"", elog(WARNING, "Backup is broken at block %u of \"%s\"",
blknum, file->path); blknum, fullpath);
return false; return false;
} }
@ -1589,7 +1589,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (header.compressed_size == PageIsTruncated) if (header.compressed_size == PageIsTruncated)
{ {
elog(LOG, "Block %u of \"%s\" is truncated", elog(LOG, "Block %u of \"%s\" is truncated",
blknum, file->path); blknum, fullpath);
continue; continue;
} }
@ -1600,7 +1600,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (read_len != MAXALIGN(header.compressed_size)) if (read_len != MAXALIGN(header.compressed_size))
{ {
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
blknum, file->path, read_len, header.compressed_size); blknum, fullpath, read_len, header.compressed_size);
return false; return false;
} }
@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
&errormsg); &errormsg);
if (uncompressed_size < 0 && errormsg != NULL) if (uncompressed_size < 0 && errormsg != NULL)
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
blknum, file->path, errormsg); blknum, fullpath, errormsg);
if (uncompressed_size != BLCKSZ) if (uncompressed_size != BLCKSZ)
{ {
@ -1630,7 +1630,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
continue; continue;
} }
elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
file->path, uncompressed_size); fullpath, uncompressed_size);
return false; return false;
} }
@ -1676,7 +1676,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (crc != file->crc) if (crc != file->crc)
{ {
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
file->path, crc, file->crc); fullpath, crc, file->crc);
is_valid = false; is_valid = false;
} }

View File

@ -1512,12 +1512,13 @@ bad_format:
*/ */
parray * parray *
dir_read_file_list(const char *root, const char *external_prefix, dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt, fio_location location) const char *file_txt, fio_location location, pg_crc32 expected_crc)
{ {
FILE *fp; FILE *fp;
parray *files; parray *files;
char buf[MAXPGPATH * 2]; char buf[BLCKSZ];
char stdio_buf[STDIO_BUFSIZE]; char stdio_buf[STDIO_BUFSIZE];
pg_crc32 content_crc = 0;
fp = fio_open_stream(file_txt, location); fp = fio_open_stream(file_txt, location);
if (fp == NULL) if (fp == NULL)
@ -1529,6 +1530,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
files = parray_new(); files = parray_new();
INIT_FILE_CRC32(true, content_crc);
while (fgets(buf, lengthof(buf), fp)) while (fgets(buf, lengthof(buf), fp))
{ {
char path[MAXPGPATH]; char path[MAXPGPATH];
@ -1546,6 +1549,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
dbOid; /* used for partial restore */ dbOid; /* used for partial restore */
pgFile *file; pgFile *file;
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
get_control_value(buf, "path", path, NULL, true); get_control_value(buf, "path", path, NULL, true);
get_control_value(buf, "size", NULL, &write_size, true); get_control_value(buf, "size", NULL, &write_size, true);
get_control_value(buf, "mode", NULL, &mode, true); get_control_value(buf, "mode", NULL, &mode, true);
@ -1598,10 +1603,21 @@ dir_read_file_list(const char *root, const char *external_prefix,
parray_append(files, file); parray_append(files, file);
} }
FIN_FILE_CRC32(true, content_crc);
if (ferror(fp)) if (ferror(fp))
elog(ERROR, "Failed to read from file: \"%s\"", file_txt); elog(ERROR, "Failed to read from file: \"%s\"", file_txt);
fio_close_stream(fp); fio_close_stream(fp);
if (expected_crc != 0 &&
expected_crc != content_crc)
{
elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u",
file_txt, content_crc, expected_crc);
return NULL;
}
return files; return files;
} }

View File

@ -565,13 +565,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
*/ */
for (i = parray_num(parent_chain) - 1; i >= 0; i--) for (i = parray_num(parent_chain) - 1; i >= 0; i--)
{ {
char control_file[MAXPGPATH];
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); backup->files = get_backup_filelist(backup, true);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
parray_qsort(backup->files, pgFileCompareRelPathWithExternal); parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
/* Set MERGING status for every member of the chain */ /* Set MERGING status for every member of the chain */

View File

@ -394,6 +394,8 @@ struct pgBackup
parray *files; /* list of files belonging to this backup parray *files; /* list of files belonging to this backup
* must be populated explicitly */ * must be populated explicitly */
char *note; char *note;
pg_crc32 content_crc;
}; };
/* Recovery target for restore and validate subcommands */ /* Recovery target for restore and validate subcommands */
@ -708,7 +710,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
PartialRestoreType partial_restore_type); PartialRestoreType partial_restore_type);
extern parray *get_backup_filelist(pgBackup *backup); extern parray *get_backup_filelist(pgBackup *backup, bool strict);
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
/* in merge.c */ /* in merge.c */
@ -867,7 +869,7 @@ extern void db_map_entry_free(void *map);
extern void print_file_list(FILE *out, const parray *files, const char *root, extern void print_file_list(FILE *out, const parray *files, const char *root,
const char *external_prefix, parray *external_list); const char *external_prefix, parray *external_list);
extern parray *dir_read_file_list(const char *root, const char *external_prefix, extern parray *dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt, fio_location location); const char *file_txt, fio_location location, pg_crc32 expected_crc);
extern parray *make_external_directory_list(const char *colon_separated_dirs, extern parray *make_external_directory_list(const char *colon_separated_dirs,
bool remap); bool remap);
extern void free_dir_list(parray *list); extern void free_dir_list(parray *list);
@ -933,7 +935,7 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
extern bool create_empty_file(fio_location from_location, const char *to_root, extern bool create_empty_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file); fio_location to_location, pgFile *file);
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version); uint32 checksum_version, uint32 backup_version);
/* parsexlog.c */ /* parsexlog.c */
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,

View File

@ -494,7 +494,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
const char *pgdata_path, bool no_sync) const char *pgdata_path, bool no_sync)
{ {
int i; int i;
char control_file[MAXPGPATH];
char timestamp[100]; char timestamp[100];
parray *dest_files = NULL; parray *dest_files = NULL;
parray *external_dirs = NULL; parray *external_dirs = NULL;
@ -515,8 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
elog(INFO, "Restoring the database from backup at %s", timestamp); elog(INFO, "Restoring the database from backup at %s", timestamp);
join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST); dest_files = get_backup_filelist(dest_backup, true);
dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
/* Lock backup chain and make sanity checks */ /* Lock backup chain and make sanity checks */
for (i = parray_num(parent_chain) - 1; i >= 0; i--) for (i = parray_num(parent_chain) - 1; i >= 0; i--)
@ -550,10 +548,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
/* populate backup filelist */ /* populate backup filelist */
if (backup->start_time != dest_backup->start_time) if (backup->start_time != dest_backup->start_time)
{ backup->files = get_backup_filelist(backup, true);
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
}
else else
backup->files = dest_files; backup->files = dest_files;
@ -1496,7 +1491,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
char database_map_path[MAXPGPATH]; char database_map_path[MAXPGPATH];
parray *files = NULL; parray *files = NULL;
files = get_backup_filelist(backup); files = get_backup_filelist(backup, true);
/* look for 'database_map' file in backup_content.control */ /* look for 'database_map' file in backup_content.control */
for (i = 0; i < parray_num(files); i++) for (i = 0; i < parray_num(files); i++)

View File

@ -430,11 +430,16 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup)
json_add_value(buf, "status", status2str(backup->status), json_level, json_add_value(buf, "status", status2str(backup->status), json_level,
true); true);
if (backup->note){ if (backup->note)
json_add_value(buf, "note", backup->note, json_add_value(buf, "note", backup->note,
json_level, true); json_level, true);
if (backup->content_crc != 0)
{
json_add_key(buf, "content-crc", json_level);
appendPQExpBuffer(buf, "%u", backup->content_crc);
} }
json_add(buf, JT_END_OBJECT, &json_level); json_add(buf, JT_END_OBJECT, &json_level);
} }

View File

@ -31,6 +31,7 @@ typedef struct
uint32 backup_version; uint32 backup_version;
BackupMode backup_mode; BackupMode backup_mode;
parray *dbOid_exclude_list; parray *dbOid_exclude_list;
const char *external_prefix;
/* /*
* Return value from the thread. * Return value from the thread.
@ -48,8 +49,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
{ {
char base_path[MAXPGPATH]; char base_path[MAXPGPATH];
char external_prefix[MAXPGPATH]; char external_prefix[MAXPGPATH];
char path[MAXPGPATH]; parray *files = NULL;
parray *files;
bool corrupted = false; bool corrupted = false;
bool validation_isok = true; bool validation_isok = true;
/* arrays with meta info for multi threaded validate */ /* arrays with meta info for multi threaded validate */
@ -110,8 +110,15 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
join_path_components(base_path, backup->root_dir, DATABASE_DIR); join_path_components(base_path, backup->root_dir, DATABASE_DIR);
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); files = get_backup_filelist(backup, false);
files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST);
if (!files)
{
elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time));
backup->status = BACKUP_STATUS_CORRUPT;
write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true);
return;
}
// if (params && params->partial_db_list) // if (params && params->partial_db_list)
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
@ -142,6 +149,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
arg->stop_lsn = backup->stop_lsn; arg->stop_lsn = backup->stop_lsn;
arg->checksum_version = backup->checksum_version; arg->checksum_version = backup->checksum_version;
arg->backup_version = parse_program_version(backup->program_version); arg->backup_version = parse_program_version(backup->program_version);
arg->external_prefix = external_prefix;
// arg->dbOid_exclude_list = dbOid_exclude_list; // arg->dbOid_exclude_list = dbOid_exclude_list;
/* By default there are some error */ /* By default there are some error */
threads_args[i].ret = 1; threads_args[i].ret = 1;
@ -223,6 +231,7 @@ pgBackupValidateFiles(void *arg)
{ {
struct stat st; struct stat st;
pgFile *file = (pgFile *) parray_get(arguments->files, i); pgFile *file = (pgFile *) parray_get(arguments->files, i);
char file_fullpath[MAXPGPATH];
if (interrupted || thread_interrupted) if (interrupted || thread_interrupted)
elog(ERROR, "Interrupted during validate"); elog(ERROR, "Interrupted during validate");
@ -244,14 +253,6 @@ pgBackupValidateFiles(void *arg)
// continue; // continue;
//} //}
/*
* Currently we don't compute checksums for
* cfs_compressed data files, so skip them.
* TODO: investigate
*/
if (file->is_cfs)
continue;
if (!pg_atomic_test_set_flag(&file->lock)) if (!pg_atomic_test_set_flag(&file->lock))
continue; continue;
@ -282,14 +283,24 @@ pgBackupValidateFiles(void *arg)
if (file->write_size == 0) if (file->write_size == 0)
continue; continue;
if (file->external_dir_num)
{
char temp[MAXPGPATH];
makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
join_path_components(file_fullpath, temp, file->rel_path);
}
else
join_path_components(file_fullpath, arguments->base_path, file->rel_path);
/* TODO: it is redundant to check file existence using stat */ /* TODO: it is redundant to check file existence using stat */
if (stat(file->path, &st) == -1) if (stat(file_fullpath, &st) == -1)
{ {
if (errno == ENOENT) if (errno == ENOENT)
elog(WARNING, "Backup file \"%s\" is not found", file->path); elog(WARNING, "Backup file \"%s\" is not found", file_fullpath);
else else
elog(WARNING, "Cannot stat backup file \"%s\": %s", elog(WARNING, "Cannot stat backup file \"%s\": %s",
file->path, strerror(errno)); file_fullpath, strerror(errno));
arguments->corrupted = true; arguments->corrupted = true;
break; break;
} }
@ -297,7 +308,7 @@ pgBackupValidateFiles(void *arg)
if (file->write_size != st.st_size) if (file->write_size != st.st_size)
{ {
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
file->path, (unsigned long) st.st_size, file->write_size); file_fullpath, (unsigned long) st.st_size, file->write_size);
arguments->corrupted = true; arguments->corrupted = true;
break; break;
} }
@ -305,8 +316,10 @@ pgBackupValidateFiles(void *arg)
/* /*
* If option skip-block-validation is set, compute only file-level CRC for * If option skip-block-validation is set, compute only file-level CRC for
* datafiles, otherwise check them block by block. * datafiles, otherwise check them block by block.
* Currently we don't compute checksums for
* cfs_compressed data files, so skip block validation for them.
*/ */
if (!file->is_datafile || skip_block_validation) if (!file->is_datafile || skip_block_validation || file->is_cfs)
{ {
/* /*
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
@ -326,14 +339,14 @@ pgBackupValidateFiles(void *arg)
!file->external_dir_num) !file->external_dir_num)
crc = get_pgcontrol_checksum(arguments->base_path); crc = get_pgcontrol_checksum(arguments->base_path);
else else
crc = pgFileGetCRC(file->path, crc = pgFileGetCRC(file_fullpath,
arguments->backup_version <= 20021 || arguments->backup_version <= 20021 ||
arguments->backup_version >= 20025, arguments->backup_version >= 20025,
false); false);
if (crc != file->crc) if (crc != file->crc)
{ {
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
file->path, crc, file->crc); file_fullpath, crc, file->crc);
arguments->corrupted = true; arguments->corrupted = true;
} }
} }
@ -344,7 +357,7 @@ pgBackupValidateFiles(void *arg)
* check page headers, checksums (if enabled) * check page headers, checksums (if enabled)
* and compute checksum of the file * and compute checksum of the file
*/ */
if (!check_file_pages(file, arguments->stop_lsn, if (!check_file_pages(file, file_fullpath, arguments->stop_lsn,
arguments->checksum_version, arguments->checksum_version,
arguments->backup_version)) arguments->backup_version))
arguments->corrupted = true; arguments->corrupted = true;