db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-24 08:52:38 +02:00
Code Issues Releases Activity

Code cleanup: validate

Browse Source
This commit is contained in:
Anastasia 2017-04-14 12:51:05 +03:00
parent 75534a00be
commit 3ca609a0ff
5 changed files with 88 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backup.c
View File

@ -516,7 +516,7 @@ do_backup(bool smooth_checkpoint)
parray_walk(files_database, pgFileFree);
parray_free(files_database);
pgBackupValidate(&current, false, false);
pgBackupValidate(&current);
return 0;
}

2
pg_probackup.c
View File

@ -28,6 +28,7 @@ char arclog_path[MAXPGPATH];
char *backup_id_string_param = NULL;
/* directory configuration */
pgBackup current;
ProbackupSubcmd backup_subcmd;
/* backup configuration */
static bool smooth_checkpoint;
@ -106,7 +107,6 @@ static pgut_option options[] =
int
main(int argc, char *argv[])
{
ProbackupSubcmd backup_subcmd;
int i;
/* initialize configuration */

5
pg_probackup.h
View File

@ -224,6 +224,7 @@ extern char arclog_path[MAXPGPATH];
/* current settings */
extern pgBackup current;
extern ProbackupSubcmd backup_subcmd;
/* exclude directory list for $PGDATA file listing */
extern const char *pgdata_exclude_dir[];
@ -286,9 +287,7 @@ extern char *slurpFile(const char *datadir,
bool safe);
/* in validate.c */
extern bool pgBackupValidate(pgBackup *backup,
bool size_only,
bool for_get_timeline);
extern void pgBackupValidate(pgBackup* backup);
extern pgBackup *read_backup(time_t timestamp);
extern void init_backup(pgBackup *backup);

3
restore.c
View File

@ -213,8 +213,7 @@ do_restore_or_validate(time_t target_backup_id,
if (backup->status == BACKUP_STATUS_OK);
{
/* TODO fix second argument */
pgBackupValidate(backup, is_restore, false);
pgBackupValidate(backup);
if (is_restore)
restore_database(backup);

183
validate.c
View File

@ -18,20 +18,15 @@ static void pgBackupValidateFiles(void *arg);
typedef struct
{
parray *files;
const char *root;
bool size_only;
bool validate_crc;
bool corrupted;
} validate_files_args;
/*
* Validate each files in the backup with its size.
* TODO: change from bool to void
* TODO: understand and probably fix second argument
* Validate backup files.
*/
bool
pgBackupValidate(pgBackup *backup,
bool size_only,
bool for_get_timeline)
void
pgBackupValidate(pgBackup *backup)
{
char *backup_id_string;
char base_path[MAXPGPATH];
@ -40,96 +35,79 @@ pgBackupValidate(pgBackup *backup,
bool corrupted = false;
pthread_t validate_threads[num_threads];
validate_files_args *validate_threads_args[num_threads];
int i;
backup_id_string = base36enc(backup->start_time);
if (!for_get_timeline)
elog(LOG, "Validate backup %s", backup_id_string);
if (backup->backup_mode != BACKUP_MODE_FULL &&
backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
elog(LOG, "Invalid backup_mode of backup %s", backup_id_string);
pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);
pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
files = dir_read_file_list(base_path, path);
/* setup threads */
for (i = 0; i < parray_num(files); i++)
{
if (backup->backup_mode == BACKUP_MODE_FULL ||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)
elog(INFO, "validate: %s backup and archive log files by %s",
backup_id_string, (size_only ? "SIZE" : "CRC"));
pgFile *file = (pgFile *) parray_get(files, i);
__sync_lock_release(&file->lock);
}
if (backup->backup_mode == BACKUP_MODE_FULL ||
backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)
/* Validate files */
for (i = 0; i < num_threads; i++)
{
int i;
elog(LOG, "database files...");
pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);
pgBackupGetPath(backup, path, lengthof(path),
DATABASE_FILE_LIST);
files = dir_read_file_list(base_path, path);
validate_files_args *arg = pg_malloc(sizeof(validate_files_args));
arg->files = files;
arg->validate_crc = true;
/* setup threads */
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
__sync_lock_release(&file->lock);
}
/* TODO Why didn't we validate checksums on restore before? */
// if (backup_subcmd == RESTORE)
// arg->validate_crc = false;
/* restore files into $PGDATA */
for (i = 0; i < num_threads; i++)
{
validate_files_args *arg = pg_malloc(sizeof(validate_files_args));
arg->files = files;
arg->root = base_path;
arg->size_only = size_only;
arg->corrupted = false;
validate_threads_args[i] = arg;
pthread_create(&validate_threads[i], NULL, (void *(*)(void *)) pgBackupValidateFiles, arg);
}
/* Wait theads */
for (i = 0; i < num_threads; i++)
{
pthread_join(validate_threads[i], NULL);
if (validate_threads_args[i]->corrupted)
corrupted = true;
pg_free(validate_threads_args[i]);
}
parray_walk(files, pgFileFree);
parray_free(files);
arg->corrupted = false;
validate_threads_args[i] = arg;
pthread_create(&validate_threads[i], NULL, (void *(*)(void *)) pgBackupValidateFiles, arg);
}
/* update status to OK */
if (corrupted)
backup->status = BACKUP_STATUS_CORRUPT;
else
backup->status = BACKUP_STATUS_OK;
/* Wait theads */
for (i = 0; i < num_threads; i++)
{
pthread_join(validate_threads[i], NULL);
if (validate_threads_args[i]->corrupted)
corrupted = true;
pg_free(validate_threads_args[i]);
}
/* cleanup */
parray_walk(files, pgFileFree);
parray_free(files);
/* Update backup status */
backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK;
pgBackupWriteConf(backup);
if (corrupted)
elog(WARNING, "backup %s is corrupted", backup_id_string);
elog(WARNING, "Backup %s is corrupted", backup_id_string);
else
elog(LOG, "backup %s is valid", backup_id_string);
return !corrupted;
}
static const char *
get_relative_path(const char *path, const char *root)
{
size_t rootlen = strlen(root);
if (strncmp(path, root, rootlen) == 0 && path[rootlen] == '/')
return path + rootlen + 1;
else
return path;
elog(LOG, "Backup %s is valid", backup_id_string);
}
/*
* Validate files in the backup with size or CRC.
* NOTE: If file is not valid, do not use ERROR log message,
* rather throw a WARNING and set arguments->corrupted = true.
* This is necessary to update backup status.
*/
static void
pgBackupValidateFiles(void *arg)
{
int i;
validate_files_args *arguments = (validate_files_args *)arg;
for (i = 0; i < parray_num(arguments->files); i++)
{
struct stat st;
@ -139,51 +117,56 @@ pgBackupValidateFiles(void *arg)
continue;
if (interrupted)
elog(ERROR, "interrupted during validate");
elog(ERROR, "Interrupted during validate");
/* skipped backup while differential backup */
/* NOTE We don't compute checksums for compressed data,
* so skip it too */
if (file->write_size == BYTES_INVALID
|| !S_ISREG(file->mode)
|| file->generation != -1)
/* Validate only regular files */
if (!S_ISREG(file->mode))
continue;
/*
* Skip files which has no data, because they
* haven't changed between backups.
*/
if (file->write_size == BYTES_INVALID)
continue;
/* We don't compute checksums for compressed data, so skip them
* TODO Add checksums to compressed files.
*/
if (file->generation != -1)
continue;
/* print progress */
elog(LOG, "(%d/%lu) %s", i + 1, (unsigned long) parray_num(arguments->files),
get_relative_path(file->path, arguments->root));
elog(LOG, "Validate files: (%d/%lu) %s",
i + 1, (unsigned long) parray_num(arguments->files), file->path);
/* always validate file size */
if (stat(file->path, &st) == -1)
{
if (errno == ENOENT)
elog(WARNING, "backup file \"%s\" vanished", file->path);
elog(WARNING, "Backup file \"%s\" is not found", file->path);
else
elog(ERROR, "cannot stat backup file \"%s\": %s",
get_relative_path(file->path, arguments->root), strerror(errno));
arguments->corrupted = true;
return;
}
if (file->write_size != st.st_size)
{
elog(WARNING, "size of backup file \"%s\" must be %lu but %lu",
get_relative_path(file->path, arguments->root),
(unsigned long) file->write_size,
(unsigned long) st.st_size);
elog(WARNING, "Cannot stat backup file \"%s\": %s",
file->path, strerror(errno));
arguments->corrupted = true;
return;
}
/* validate CRC too */
if (!arguments->size_only)
if (file->write_size != st.st_size)
{
elog(WARNING, "Invalid size of backup file \"%s\" : %lu. Expected %lu",
file->path, (unsigned long) file->write_size,
(unsigned long) st.st_size);
arguments->corrupted = true;
return;
}
if (arguments->validate_crc)
{
pg_crc32 crc;
crc = pgFileGetCRC(file);
if (crc != file->crc)
{
elog(WARNING, "CRC of backup file \"%s\" must be %X but %X",
get_relative_path(file->path, arguments->root), file->crc, crc);
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
file->path, file->crc, crc);
arguments->corrupted = true;
return;
}