1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2024-11-24 08:52:38 +02:00

[Issue #66] add sanity checks for incremental restore

This commit is contained in:
Grigory Smolkin 2020-05-28 01:30:57 +03:00
parent 79c8ea3dab
commit 87252a82eb
8 changed files with 246 additions and 32 deletions

View File

@ -1902,7 +1902,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
if (file->linked)
len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
if (file->n_blocks != BLOCKNUM_INVALID)
if (file->n_blocks > 0)
len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
sprintf(line+len, "}\n");

View File

@ -1021,7 +1021,7 @@ opt_externaldir_map(ConfigOption *opt, const char *arg)
*/
void
create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir,
bool extract_tablespaces, fio_location location)
bool extract_tablespaces, bool incremental, fio_location location)
{
int i;
parray *links = NULL;
@ -1126,7 +1126,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
fio_mkdir(linked_path, pg_tablespace_mode, location);
/* create link to linked_path */
if (fio_symlink(linked_path, to_path, location) < 0)
if (fio_symlink(linked_path, to_path, incremental, location) < 0)
elog(ERROR, "Could not create symbolic link \"%s\": %s",
to_path, strerror(errno));
@ -1203,13 +1203,18 @@ read_tablespace_map(parray *files, const char *backup_dir)
/*
* Check that all tablespace mapping entries have correct linked directory
* paths. Linked directories must be empty or do not exist.
* paths. Linked directories must be empty or do not exist, unless
* we are running incremental restore, then linked directories can be nonempty.
*
* If tablespace-mapping option is supplied, all OLDDIR entries must have
* entries in tablespace_map file.
*
* When running incremental restore with tablespace remapping, then
* new tablespace directory MUST be empty, because there is no
* we can be sure, that files laying there belong to our instance.
*/
void
check_tablespace_mapping(pgBackup *backup)
check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty)
{
// char this_backup_path[MAXPGPATH];
parray *links;
@ -1236,6 +1241,18 @@ check_tablespace_mapping(pgBackup *backup)
elog(ERROR, "--tablespace-mapping option's old directory "
"doesn't have an entry in tablespace_map file: \"%s\"",
cell->old_dir);
/* For incremental restore, check that new directory is empty */
if (incremental)
{
if (!is_absolute_path(cell->new_dir))
elog(ERROR, "tablespace directory is not an absolute path: %s\n",
cell->new_dir);
if (!dir_is_empty(cell->new_dir, FIO_DB_HOST))
elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
cell->new_dir);
}
}
/* 2 - all linked directories must be empty */
@ -1257,8 +1274,12 @@ check_tablespace_mapping(pgBackup *backup)
linked_path);
if (!dir_is_empty(linked_path, FIO_DB_HOST))
elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
linked_path);
{
if (!incremental)
elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
linked_path);
*tblspaces_are_empty = false;
}
}
free(tmp_file);

View File

@ -589,7 +589,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
/* Create directories */
create_data_directories(dest_backup->files, full_database_dir,
dest_backup->root_dir, false, FIO_BACKUP_HOST);
dest_backup->root_dir, false, false, FIO_BACKUP_HOST);
/* External directories stuff */
if (dest_backup->external_dir_str)

View File

@ -867,12 +867,13 @@ extern void create_data_directories(parray *dest_files,
const char *data_dir,
const char *backup_dir,
bool extract_tablespaces,
bool incremental,
fio_location location);
extern void read_tablespace_map(parray *files, const char *backup_dir);
extern void opt_tablespace_map(ConfigOption *opt, const char *arg);
extern void opt_externaldir_map(ConfigOption *opt, const char *arg);
extern void check_tablespace_mapping(pgBackup *backup);
extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty);
extern void check_external_dir_mapping(pgBackup *backup);
extern char *get_external_remap(char *current_dir);
@ -951,6 +952,7 @@ extern bool create_empty_file(fio_location from_location, const char *to_root,
extern uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
extern pid_t check_postmaster(const char *pgdata);
extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version);
@ -1048,6 +1050,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
extern uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
extern pid_t fio_check_postmaster(const char *pgdata);
extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg);

View File

@ -49,6 +49,8 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include);
static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray *dbOid_exclude_list, pgRestoreParams *params,
const char *pgdata_path, bool no_sync);
static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
bool lsn_based);
/*
* Iterate over backup list to find all ancestors of the broken parent_backup
@ -111,6 +113,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
char *action = params->is_restore ? "Restore":"Validate";
parray *parent_chain = NULL;
parray *dbOid_exclude_list = NULL;
bool pgdata_is_empty = true;
bool tblspaces_are_empty = true;
if (params->is_restore)
{
@ -126,16 +130,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
{
elog(INFO, "Running incremental restore into nonempty directory: \"%s\"",
instance_config.pgdata);
check_incremental_compatibility(instance_config.pgdata,
instance_config.system_identifier,
false);
}
else
elog(ERROR, "Restore destination is not empty: \"%s\"",
instance_config.pgdata);
}
else
{
/* if remote directory is empty then disable incremental restore */
if (params->incremental)
params->incremental = false;
/* if remote directory is empty then incremental restore may be disabled */
pgdata_is_empty = true;
}
}
@ -340,7 +345,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
*/
if (params->is_restore)
{
check_tablespace_mapping(dest_backup);
check_tablespace_mapping(dest_backup, params->incremental, &tblspaces_are_empty);
if (pgdata_is_empty && tblspaces_are_empty)
params->incremental = false;
/* no point in checking external directories if their restore is not requested */
if (!params->skip_external_dirs)
@ -602,7 +610,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
* Restore dest_backup internal directories.
*/
create_data_directories(dest_files, instance_config.pgdata,
dest_backup->root_dir, true, FIO_DB_HOST);
dest_backup->root_dir, true,
params->incremental, FIO_DB_HOST);
/*
* Restore dest_backup external directories.
@ -660,16 +669,23 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
elog(INFO, "Extracting the content of destination directory for incremental restore");
/* TODO: external directorues */
time(&start_time);
if (fio_is_remote(FIO_DB_HOST))
fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0);
else
dir_list_file(pgdata_files, pgdata_path,
false, true, false, false, true, 0, FIO_LOCAL_HOST);
parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc);
elog(INFO, "Destination directory content extracted, time elapsed:");
elog(INFO, "Removing redundant files");
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
elog(INFO, "Destination directory content extracted, time elapsed: %s",
pretty_time);
elog(INFO, "Removing redundant files in destination directory");
time(&start_time);
for (i = 0; i < parray_num(pgdata_files); i++)
{
pgFile *file = (pgFile *) parray_get(pgdata_files, i);
@ -686,7 +702,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
}
}
elog(INFO, "Redundant files are removed, time elapsed:");
time(&end_time);
pretty_time_interval(difftime(end_time, start_time),
pretty_time, lengthof(pretty_time));
elog(INFO, "Redundant files are removed, time elapsed: %s", pretty_time);
// use_bitmap = true;
/* At this point PDATA do not contain files, that also do not exists in backup filelist */
@ -756,9 +776,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s",
pretty_total_bytes, pretty_time);
elog(INFO, "Approximate restore efficiency ratio: %.f%% (%s/%s)",
((float) dest_bytes / total_bytes) * 100,
pretty_dest_bytes, pretty_total_bytes);
elog(INFO, "Restore overwriting ratio (less is better): %.f%% (%s/%s)",
((float) total_bytes / dest_bytes) * 100,
pretty_total_bytes, pretty_dest_bytes);
}
else
elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s",
@ -921,7 +941,9 @@ restore_files(void *arg)
if (arguments->incremental &&
parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc))
{
already_exists = true;
}
/*
* Handle incremental restore case for data files.
@ -933,7 +955,6 @@ restore_files(void *arg)
dest_file->is_datafile && !dest_file->is_cfs &&
dest_file->n_blocks > 0)
{
elog(INFO, "HELLO");
/* remote mode */
if (fio_is_remote(FIO_DB_HOST))
checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version,
@ -1743,3 +1764,72 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
return dbOid_exclude_list;
}
/* check that instance has the same SYSTEM_ID, */
void
check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool lsn_based)
{
uint64 system_id_pgdata;
bool success = true;
pid_t pid;
char backup_label[MAXPGPATH];
/* slurp pg_control and check that system ID is the same */
/* check that instance is not running */
/* if lsn_based, check that there is no backup_label files is around AND
* get redo point lsn from destination pg_control.
* It is really important to be sure that pg_control is in cohesion with
* data files content, because based on pg_control information we will
* choose a backup suitable for lsn based incremental restore.
*/
/* TODO: handle timeline discrepancies */
system_id_pgdata = get_system_identifier(pgdata);
if (system_id_pgdata != instance_config.system_identifier)
{
elog(WARNING, "Backup catalog was initialized for system id %lu, "
"but destination directory system id is %lu",
system_identifier, system_id_pgdata);
success = false;
}
/* check postmaster pid */
if (fio_is_remote(FIO_DB_HOST))
pid = fio_check_postmaster(pgdata);
else
pid = check_postmaster(pgdata);
if (pid == 1) /* postmaster.pid is mangled */
{
char pid_file[MAXPGPATH];
snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata);
elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not",
pid_file);
success = false;
}
else if (pid > 1)
{
elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"",
pid, pgdata);
success = false;
}
if (lsn_based)
{
snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata);
if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0)
{
elog(WARNING, "Destination directory contains \"backup_control\" file. "
"It does not mean that you should delete this file, only that "
"lsn-based incremental restore is dangerous to use in this case. "
"Consider to use checksum-based incremental restore");
success = false;
}
}
if (!success)
elog(ERROR, "Incremental restore is impossible");
}

View File

@ -549,3 +549,50 @@ datapagemap_print_debug(datapagemap_t *map)
pg_free(iter);
}
/*
* return pid of postmaster process running in given pgdata.
* return 0 if there is none.
*/
pid_t
check_postmaster(const char *pgdata)
{
FILE *fp;
pid_t pid;
char pid_file[MAXPGPATH];
snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata);
fp = fopen(pid_file, "r");
if (fp == NULL)
{
/* No pid file, acceptable*/
if (errno == ENOENT)
return 0;
else
elog(ERROR, "Cannot open file \"%s\": %s",
pid_file, strerror(errno));
}
if (fscanf(fp, "%i", &pid) != 1)
{
/* something is wrong with the file content */
pid = 1;
}
if (pid > 1)
{
if (kill(pid, 0) != 0)
{
/* process no longer exists */
if (errno == ESRCH)
pid = 0;
else
elog(ERROR, "Failed to send signal 0 to a process %d: %s",
pid, strerror(errno));
}
}
fclose(fp);
return pid;
}

View File

@ -856,7 +856,7 @@ int fio_access(char const* path, int mode, fio_location location)
}
/* Create symbolic link */
int fio_symlink(char const* target, char const* link_path, fio_location location)
int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location)
{
if (fio_is_remote(location))
{
@ -866,6 +866,7 @@ int fio_symlink(char const* target, char const* link_path, fio_location location
hdr.cop = FIO_SYMLINK;
hdr.handle = -1;
hdr.size = target_len + link_path_len;
hdr.arg = overwrite ? 1 : 0;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len);
@ -875,10 +876,26 @@ int fio_symlink(char const* target, char const* link_path, fio_location location
}
else
{
if (overwrite)
remove_file_or_dir(link_path);
return symlink(target, link_path);
}
}
static void fio_symlink_impl(int out, char *buf, bool overwrite)
{
char *linked_path = buf;
char *link_path = buf + strlen(buf) + 1;
if (overwrite)
remove_file_or_dir(link_path);
if (symlink(linked_path, link_path))
elog(ERROR, "Could not create symbolic link \"%s\": %s",
link_path, strerror(errno));
}
/* Rename file */
int fio_rename(char const* old_path, char const* new_path, fio_location location)
{
@ -1394,6 +1411,9 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt
--------------------------------------------------------------
*/
// elog(WARNING, "Size: %lu", sizeof(fio_header));
// elog(ERROR, "Size: %lu", MAXALIGN(sizeof(fio_header)));
req.hdr.cop = FIO_SEND_PAGES;
if (pagemap)
@ -2243,7 +2263,6 @@ static void fio_list_dir_impl(int out, char* buf)
uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)
{
fio_header hdr;
fio_checksum_map_request req_hdr;
uint16 *checksum_map = NULL;
@ -2293,6 +2312,34 @@ static void fio_get_checksum_map_impl(int out, char *buf)
pg_free(checksum_map);
}
pid_t fio_check_postmaster(const char *pgdata)
{
fio_header hdr;
hdr.cop = FIO_CHECK_POSTMASTER;
hdr.size = strlen(pgdata) + 1;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size);
/* receive result */
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
return hdr.arg;
}
static void fio_check_postmaster_impl(int out, char *buf)
{
fio_header hdr;
pid_t postmaster_pid;
char *pgdata = (char*) buf;
postmaster_pid = check_postmaster(pgdata);
/* send arrays of checksums to main process */
hdr.arg = postmaster_pid;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
/* Execute commands at remote host */
void fio_communicate(int in, int out)
{
@ -2411,7 +2458,7 @@ void fio_communicate(int in, int out)
SYS_CHECK(rename(buf, buf + strlen(buf) + 1));
break;
case FIO_SYMLINK: /* Create symbolic link */
SYS_CHECK(symlink(buf, buf + strlen(buf) + 1));
fio_symlink_impl(out, buf, hdr.arg > 0 ? true : false);
break;
case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */
SYS_CHECK(remove_file_or_dir(buf));
@ -2468,6 +2515,10 @@ void fio_communicate(int in, int out)
/* calculate crc32 for a file */
fio_get_checksum_map_impl(out, buf);
break;
case FIO_CHECK_POSTMASTER:
/* calculate crc32 for a file */
fio_check_postmaster_impl(out, buf);
break;
case FIO_DISCONNECT:
hdr.cop = FIO_SEND_FILE_EOF;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));

View File

@ -47,7 +47,8 @@ typedef enum
FIO_DISCONNECT,
/* message for compatibility check */
FIO_AGENT_VERSION,
FIO_LIST_DIR
FIO_LIST_DIR,
FIO_CHECK_POSTMASTER
} fio_operations;
typedef enum
@ -66,8 +67,9 @@ typedef enum
typedef struct
{
unsigned cop : 5;
unsigned handle : 7;
// fio_operations cop;
unsigned cop : 6;
unsigned handle : 6;
unsigned size : 20;
unsigned arg;
} fio_header;
@ -107,7 +109,7 @@ extern int fio_sync(char const* path, fio_location location);
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress);
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
extern int fio_symlink(char const* target, char const* link_path, fio_location location);
extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);
extern int fio_unlink(char const* path, fio_location location);
extern int fio_mkdir(char const* path, int mode, fio_location location);
extern int fio_chmod(char const* path, int mode, fio_location location);