1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-07 13:40:17 +02:00

[Issue #203] Add new pgBackup attribute 'content-crc', containing crc of backup_content.control file, to detect corruption

This commit is contained in:
Grigory Smolkin 2020-05-11 15:06:07 +03:00
parent e368eb7380
commit 9b0922c6b7
9 changed files with 127 additions and 86 deletions

View File

@ -217,8 +217,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
if (prev_backup)
{
char prev_backup_filelist_path[MAXPGPATH];
if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION))
elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
"pg_probackup do not guarantee to be forward compatible. "
@ -227,10 +225,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
DATABASE_FILE_LIST);
/* Files of previous backup needed by DELTA backup */
prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST);
prev_backup_filelist = get_backup_filelist(prev_backup, true);
/* If lsn is not NULL, only pages with higher lsn will be copied. */
prev_backup_start_lsn = prev_backup->start_lsn;

View File

@ -538,17 +538,17 @@ err_proc:
* TODO this function only used once. Is it really needed?
*/
parray *
get_backup_filelist(pgBackup *backup)
get_backup_filelist(pgBackup *backup, bool strict)
{
parray *files = NULL;
char backup_filelist_path[MAXPGPATH];
join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST);
files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc);
/* redundant sanity? */
if (!files)
elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time));
elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time));
return files;
}
@ -1759,6 +1759,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
if (backup->note)
fio_fprintf(out, "note = '%s'\n", backup->note);
if (backup->content_crc != 0)
fio_fprintf(out, "content-crc = %u\n", backup->content_crc);
}
/*
@ -1813,8 +1816,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
parray *external_list, bool sync)
{
FILE *out;
char path[MAXPGPATH];
char path_temp[MAXPGPATH];
char control_path[MAXPGPATH];
char control_path_temp[MAXPGPATH];
size_t i = 0;
#define BUFFERSZ 1024*1024
char *buf;
@ -1822,24 +1825,29 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
int64 uncompressed_size_on_disk = 0;
int64 wal_size_on_disk = 0;
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
snprintf(path_temp, sizeof(path_temp), "%s.tmp", path);
join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST);
snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path);
out = fopen(path_temp, PG_BINARY_W);
out = fopen(control_path_temp, PG_BINARY_W);
if (out == NULL)
elog(ERROR, "Cannot open file list \"%s\": %s", path_temp,
elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp,
strerror(errno));
if (chmod(path_temp, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp,
if (chmod(control_path_temp, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp,
strerror(errno));
buf = pgut_malloc(BUFFERSZ);
setvbuf(out, buf, _IOFBF, BUFFERSZ);
if (sync)
INIT_FILE_CRC32(true, backup->content_crc);
/* print each file in the list */
for (i = 0; i < parray_num(files); i++)
{
int len = 0;
char line[BLCKSZ];
pgFile *file = (pgFile *) parray_get(files, i);
char *path = file->path; /* for streamed WAL files */
@ -1873,7 +1881,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
(file->external_dir_num && external_list))
path = file->rel_path;
fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
@ -1887,32 +1895,40 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
file->dbOid);
if (file->is_datafile)
fprintf(out, ",\"segno\":\"%d\"", file->segno);
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
if (file->linked)
fprintf(out, ",\"linked\":\"%s\"", file->linked);
len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
if (file->n_blocks != BLOCKNUM_INVALID)
fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks);
len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
fprintf(out, "}\n");
sprintf(line+len, "}\n");
if (sync)
COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line));
fprintf(out, "%s", line);
}
if (sync)
FIN_FILE_CRC32(true, backup->content_crc);
if (fflush(out) != 0)
elog(ERROR, "Cannot flush file list \"%s\": %s",
path_temp, strerror(errno));
control_path_temp, strerror(errno));
if (sync && fsync(fileno(out)) < 0)
elog(ERROR, "Cannot sync file list \"%s\": %s",
path_temp, strerror(errno));
control_path_temp, strerror(errno));
if (fclose(out) != 0)
elog(ERROR, "Cannot close file list \"%s\": %s",
path_temp, strerror(errno));
control_path_temp, strerror(errno));
if (rename(path_temp, path) < 0)
if (rename(control_path_temp, control_path) < 0)
elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s",
path_temp, path, strerror(errno));
control_path_temp, control_path, strerror(errno));
/* use extra variable to avoid reset of previous data_bytes value in case of error */
backup->data_bytes = backup_size_on_disk;
@ -1975,6 +1991,7 @@ readBackupControlFile(const char *path)
{'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT},
{'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT},
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
{'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT},
{0}
};
@ -2242,6 +2259,7 @@ pgBackupInit(pgBackup *backup)
backup->root_dir = NULL;
backup->files = NULL;
backup->note = NULL;
backup->content_crc = 0;
}

View File

@ -1462,7 +1462,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
*/
if (errno == ENOENT)
{
elog(LOG, "File \"%s\" is not found", file->path);
elog(LOG, "File \"%s\" is not found", from_fullpath);
return true;
}
@ -1472,7 +1472,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
}
if (file->size % BLCKSZ != 0)
elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size);
/*
* Compute expected number of blocks in the file.
@ -1508,8 +1508,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
/* Valiate pages of datafile in backup one by one */
bool
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
uint32 backup_version)
check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version)
{
size_t read_len = 0;
bool is_valid = true;
@ -1517,19 +1517,19 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
pg_crc32 crc;
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path);
elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath);
in = fopen(file->path, PG_BINARY_R);
in = fopen(fullpath, PG_BINARY_R);
if (in == NULL)
{
if (errno == ENOENT)
{
elog(WARNING, "File \"%s\" is not found", file->path);
elog(WARNING, "File \"%s\" is not found", fullpath);
return false;
}
elog(ERROR, "Cannot open file \"%s\": %s",
file->path, strerror(errno));
fullpath, strerror(errno));
}
/* calc CRC of backup file */
@ -1553,7 +1553,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (ferror(in))
elog(ERROR, "Cannot read header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno));
blknum, fullpath, strerror(errno));
if (read_len != sizeof(header))
{
@ -1562,10 +1562,10 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
else if (read_len != 0 && feof(in))
elog(WARNING,
"Odd size page found at block %u of \"%s\"",
blknum, file->path);
blknum, fullpath);
else
elog(WARNING, "Cannot read header of block %u of \"%s\": %s",
blknum, file->path, strerror(errno));
blknum, fullpath, strerror(errno));
return false;
}
@ -1573,14 +1573,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (header.block == 0 && header.compressed_size == 0)
{
elog(VERBOSE, "Skip empty block of \"%s\"", file->path);
elog(VERBOSE, "Skip empty block of \"%s\"", fullpath);
continue;
}
if (header.block < blknum)
{
elog(WARNING, "Backup is broken at block %u of \"%s\"",
blknum, file->path);
blknum, fullpath);
return false;
}
@ -1589,7 +1589,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (header.compressed_size == PageIsTruncated)
{
elog(LOG, "Block %u of \"%s\" is truncated",
blknum, file->path);
blknum, fullpath);
continue;
}
@ -1600,7 +1600,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (read_len != MAXALIGN(header.compressed_size))
{
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
blknum, file->path, read_len, header.compressed_size);
blknum, fullpath, read_len, header.compressed_size);
return false;
}
@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
&errormsg);
if (uncompressed_size < 0 && errormsg != NULL)
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
blknum, file->path, errormsg);
blknum, fullpath, errormsg);
if (uncompressed_size != BLCKSZ)
{
@ -1630,7 +1630,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
continue;
}
elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ",
file->path, uncompressed_size);
fullpath, uncompressed_size);
return false;
}
@ -1676,7 +1676,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X",
file->path, crc, file->crc);
fullpath, crc, file->crc);
is_valid = false;
}

View File

@ -1512,12 +1512,13 @@ bad_format:
*/
parray *
dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt, fio_location location)
const char *file_txt, fio_location location, pg_crc32 expected_crc)
{
FILE *fp;
parray *files;
char buf[MAXPGPATH * 2];
char stdio_buf[STDIO_BUFSIZE];
FILE *fp;
parray *files;
char buf[BLCKSZ];
char stdio_buf[STDIO_BUFSIZE];
pg_crc32 content_crc = 0;
fp = fio_open_stream(file_txt, location);
if (fp == NULL)
@ -1529,6 +1530,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
files = parray_new();
INIT_FILE_CRC32(true, content_crc);
while (fgets(buf, lengthof(buf), fp))
{
char path[MAXPGPATH];
@ -1546,6 +1549,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
dbOid; /* used for partial restore */
pgFile *file;
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
get_control_value(buf, "path", path, NULL, true);
get_control_value(buf, "size", NULL, &write_size, true);
get_control_value(buf, "mode", NULL, &mode, true);
@ -1598,10 +1603,21 @@ dir_read_file_list(const char *root, const char *external_prefix,
parray_append(files, file);
}
FIN_FILE_CRC32(true, content_crc);
if (ferror(fp))
elog(ERROR, "Failed to read from file: \"%s\"", file_txt);
fio_close_stream(fp);
if (expected_crc != 0 &&
expected_crc != content_crc)
{
elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u",
file_txt, content_crc, expected_crc);
return NULL;
}
return files;
}

View File

@ -565,13 +565,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
*/
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
{
char control_file[MAXPGPATH];
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
backup->files = get_backup_filelist(backup, true);
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
/* Set MERGING status for every member of the chain */

View File

@ -394,6 +394,8 @@ struct pgBackup
parray *files; /* list of files belonging to this backup
* must be populated explicitly */
char *note;
pg_crc32 content_crc;
};
/* Recovery target for restore and validate subcommands */
@ -708,7 +710,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
PartialRestoreType partial_restore_type);
extern parray *get_backup_filelist(pgBackup *backup);
extern parray *get_backup_filelist(pgBackup *backup, bool strict);
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
/* in merge.c */
@ -867,7 +869,7 @@ extern void db_map_entry_free(void *map);
extern void print_file_list(FILE *out, const parray *files, const char *root,
const char *external_prefix, parray *external_list);
extern parray *dir_read_file_list(const char *root, const char *external_prefix,
const char *file_txt, fio_location location);
const char *file_txt, fio_location location, pg_crc32 expected_crc);
extern parray *make_external_directory_list(const char *colon_separated_dirs,
bool remap);
extern void free_dir_list(parray *list);
@ -933,7 +935,7 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
extern bool create_empty_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file);
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version);
/* parsexlog.c */
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,

View File

@ -494,7 +494,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
const char *pgdata_path, bool no_sync)
{
int i;
char control_file[MAXPGPATH];
char timestamp[100];
parray *dest_files = NULL;
parray *external_dirs = NULL;
@ -515,8 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
time2iso(timestamp, lengthof(timestamp), dest_backup->start_time);
elog(INFO, "Restoring the database from backup at %s", timestamp);
join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST);
dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
dest_files = get_backup_filelist(dest_backup, true);
/* Lock backup chain and make sanity checks */
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
@ -550,10 +548,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
/* populate backup filelist */
if (backup->start_time != dest_backup->start_time)
{
join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST);
backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST);
}
backup->files = get_backup_filelist(backup, true);
else
backup->files = dest_files;
@ -1496,7 +1491,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
char database_map_path[MAXPGPATH];
parray *files = NULL;
files = get_backup_filelist(backup);
files = get_backup_filelist(backup, true);
/* look for 'database_map' file in backup_content.control */
for (i = 0; i < parray_num(files); i++)

View File

@ -430,11 +430,16 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup)
json_add_value(buf, "status", status2str(backup->status), json_level,
true);
if (backup->note){
if (backup->note)
json_add_value(buf, "note", backup->note,
json_level, true);
if (backup->content_crc != 0)
{
json_add_key(buf, "content-crc", json_level);
appendPQExpBuffer(buf, "%u", backup->content_crc);
}
json_add(buf, JT_END_OBJECT, &json_level);
}

View File

@ -31,6 +31,7 @@ typedef struct
uint32 backup_version;
BackupMode backup_mode;
parray *dbOid_exclude_list;
const char *external_prefix;
/*
* Return value from the thread.
@ -48,8 +49,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
{
char base_path[MAXPGPATH];
char external_prefix[MAXPGPATH];
char path[MAXPGPATH];
parray *files;
parray *files = NULL;
bool corrupted = false;
bool validation_isok = true;
/* arrays with meta info for multi threaded validate */
@ -110,8 +110,15 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
join_path_components(base_path, backup->root_dir, DATABASE_DIR);
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST);
files = get_backup_filelist(backup, false);
if (!files)
{
elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time));
backup->status = BACKUP_STATUS_CORRUPT;
write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true);
return;
}
// if (params && params->partial_db_list)
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
@ -142,6 +149,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
arg->stop_lsn = backup->stop_lsn;
arg->checksum_version = backup->checksum_version;
arg->backup_version = parse_program_version(backup->program_version);
arg->external_prefix = external_prefix;
// arg->dbOid_exclude_list = dbOid_exclude_list;
/* By default there are some error */
threads_args[i].ret = 1;
@ -223,6 +231,7 @@ pgBackupValidateFiles(void *arg)
{
struct stat st;
pgFile *file = (pgFile *) parray_get(arguments->files, i);
char file_fullpath[MAXPGPATH];
if (interrupted || thread_interrupted)
elog(ERROR, "Interrupted during validate");
@ -244,14 +253,6 @@ pgBackupValidateFiles(void *arg)
// continue;
//}
/*
* Currently we don't compute checksums for
* cfs_compressed data files, so skip them.
* TODO: investigate
*/
if (file->is_cfs)
continue;
if (!pg_atomic_test_set_flag(&file->lock))
continue;
@ -282,14 +283,24 @@ pgBackupValidateFiles(void *arg)
if (file->write_size == 0)
continue;
if (file->external_dir_num)
{
char temp[MAXPGPATH];
makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
join_path_components(file_fullpath, temp, file->rel_path);
}
else
join_path_components(file_fullpath, arguments->base_path, file->rel_path);
/* TODO: it is redundant to check file existence using stat */
if (stat(file->path, &st) == -1)
if (stat(file_fullpath, &st) == -1)
{
if (errno == ENOENT)
elog(WARNING, "Backup file \"%s\" is not found", file->path);
elog(WARNING, "Backup file \"%s\" is not found", file_fullpath);
else
elog(WARNING, "Cannot stat backup file \"%s\": %s",
file->path, strerror(errno));
file_fullpath, strerror(errno));
arguments->corrupted = true;
break;
}
@ -297,7 +308,7 @@ pgBackupValidateFiles(void *arg)
if (file->write_size != st.st_size)
{
elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
file->path, (unsigned long) st.st_size, file->write_size);
file_fullpath, (unsigned long) st.st_size, file->write_size);
arguments->corrupted = true;
break;
}
@ -305,8 +316,10 @@ pgBackupValidateFiles(void *arg)
/*
* If option skip-block-validation is set, compute only file-level CRC for
* datafiles, otherwise check them block by block.
* Currently we don't compute checksums for
* cfs_compressed data files, so skip block validation for them.
*/
if (!file->is_datafile || skip_block_validation)
if (!file->is_datafile || skip_block_validation || file->is_cfs)
{
/*
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
@ -326,14 +339,14 @@ pgBackupValidateFiles(void *arg)
!file->external_dir_num)
crc = get_pgcontrol_checksum(arguments->base_path);
else
crc = pgFileGetCRC(file->path,
crc = pgFileGetCRC(file_fullpath,
arguments->backup_version <= 20021 ||
arguments->backup_version >= 20025,
false);
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
file->path, crc, file->crc);
file_fullpath, crc, file->crc);
arguments->corrupted = true;
}
}
@ -344,7 +357,7 @@ pgBackupValidateFiles(void *arg)
* check page headers, checksums (if enabled)
* and compute checksum of the file
*/
if (!check_file_pages(file, arguments->stop_lsn,
if (!check_file_pages(file, file_fullpath, arguments->stop_lsn,
arguments->checksum_version,
arguments->backup_version))
arguments->corrupted = true;