db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

[Issue #228] fix merge

Browse Source
This commit is contained in:
Grigory Smolkin
2020-06-19 02:32:43 +03:00
parent cb7050320b
commit d9fbced324
5 changed files with 79 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backup.c
+1 -1
View File
@@ -2161,7 +2161,7 @@ backup_files(void *arg)
arguments->nodeInfo->checksum_version,
arguments->nodeInfo->ptrack_version_num,
arguments->nodeInfo->ptrack_schema,
arguments->hdr_map, true);
arguments->hdr_map, false);
}
else
{
src/catalog.c
+7 -1
View File
@@ -474,9 +474,11 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
backup->database_dir = pgut_malloc(MAXPGPATH);
join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR);
/* block header map */
/* block header map, TODO: move to separate function */
backup->hdr_map.path = pgut_malloc(MAXPGPATH);
join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP);
backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH);
join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP);
backup->hdr_map.fp = NULL;
/* TODO: save encoded backup id */
@@ -855,6 +857,8 @@ pgBackupCreateDir(pgBackup *backup)
/* block header map */
backup->hdr_map.path = pgut_malloc(MAXPGPATH);
join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP);
backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH);
join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP);
backup->hdr_map.fp = NULL;
/* create directories for actual backup files */
@@ -2282,6 +2286,7 @@ pgBackupInit(pgBackup *backup)
backup->content_crc = 0;
backup->hdr_map.path = NULL;
backup->hdr_map.path_tmp = NULL;
backup->hdr_map.fp = NULL;
backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
}
@@ -2298,6 +2303,7 @@ pgBackupFree(void *backup)
pg_free(b->database_dir);
pg_free(b->note);
pg_free(b->hdr_map.path);
pg_free(b->hdr_map.path_tmp);
pg_free(backup);
}
src/data.c
+26 -15
View File
@@ -31,8 +31,6 @@ typedef struct DataPage
char data[BLCKSZ];
} DataPage;
static BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version);
static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map);
static bool get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph,
pg_crc32 *crc, bool use_crc32c);
@@ -538,7 +536,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
CompressAlg calg, int clevel, uint32 checksum_version,
int ptrack_version_num, const char *ptrack_schema,
HeaderMap *hdr_map, bool missing_ok)
HeaderMap *hdr_map, bool is_merge)
{
int rc;
bool use_pagemap;
@@ -629,7 +627,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
/* check for errors */
if (rc == FILE_MISSING)
{
elog(LOG, "File \"%s\" is not found", from_fullpath);
elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath);
file->write_size = FILE_NOT_FOUND;
goto cleanup;
}
@@ -685,7 +683,7 @@ cleanup:
FIN_FILE_CRC32(true, file->crc);
/* dump page headers */
write_page_headers(headers, file, hdr_map);
write_page_headers(headers, file, hdr_map, is_merge);
pg_free(errmsg);
pg_free(file->pagemap.bitmap);
@@ -818,8 +816,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
/* set stdio buffering for input data file */
setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE);
// elog(INFO, "N_HEADERS: %i", tmp_file->n_headers);
// elog(INFO, "File: %s", tmp_file->rel_path);
// elog(INFO, "Backup: %s", base36enc(backup->start_time));
/* get headers for this file */
if (use_headers)
if (use_headers && tmp_file->n_headers > 0)
headers = get_data_file_headers(&(backup->hdr_map), tmp_file,
parse_program_version(backup->program_version));
@@ -2163,8 +2165,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
FIN_FILE_CRC32(true, hdr_crc);
if (hdr_crc != file->hdr_crc)
elog(ERROR, "Header file crc mismatch \"%s\", current: %u, expected: %u",
hdr_map->path, hdr_crc, file->hdr_crc);
elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u",
file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc);
if (fclose(in))
elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno));
@@ -2173,29 +2175,35 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
}
void
write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map)
write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge)
{
size_t read_len = 0;
size_t read_len = 0;
char *map_path = NULL;
if (file->n_headers <= 0)
return;
/* when running merge we must save headers into the temp map */
map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path;
/* writing to header map must be serialized */
pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */
if (!hdr_map->fp)
{
hdr_map->fp = fopen(hdr_map->path, PG_BINARY_W);
elog(LOG, "Creating page header map \"%s\"", map_path);
hdr_map->fp = fopen(map_path, PG_BINARY_W);
if (hdr_map->fp == NULL)
elog(ERROR, "Cannot open header file \"%s\": %s",
hdr_map->path, strerror(errno));
map_path, strerror(errno));
/* disable buffering for header file */
setvbuf(hdr_map->fp, NULL, _IONBF, BUFSIZ);
/* update file permission */
if (chmod(hdr_map->path, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", hdr_map->path,
if (chmod(map_path, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", map_path,
strerror(errno));
file->hdr_off = 0;
@@ -2211,7 +2219,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map)
FIN_FILE_CRC32(true, file->hdr_crc);
if (fwrite(headers, 1, read_len, hdr_map->fp) != read_len)
elog(ERROR, "Cannot write to file \"%s\": %s", hdr_map->path, strerror(errno));
elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno));
elog(VERBOSE, "Writing header map for file \"%s\" offset: %lu, len: %lu, crc: %u",
file->rel_path, file->hdr_off, read_len, file->hdr_crc);
pthread_mutex_unlock(&(hdr_map->mutex));
}
src/merge.c
+40 -6
View File
@@ -550,11 +550,11 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
elog(INFO, "Validate parent chain for backup %s",
base36enc(dest_backup->start_time));
/* forbid merge retry for failed merges between 2.4.0 and any
/* Forbid merge retry for failed merges between 2.4.0 and any
* older version. Several format changes makes it impossible
* to determine the exact format any speific file is got.
*/
if (full_backup->status == BACKUP_STATUS_MERGING &&
if (is_retry &&
parse_program_version(dest_backup->program_version) >= 20400 &&
parse_program_version(full_backup->program_version) < 20400)
{
@@ -705,6 +705,23 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
elog(ERROR, "Backup files merging failed, time elapsed: %s",
pretty_time);
/* If temp header map descriptor is open, then close it and make rename */
if (full_backup->hdr_map.fp)
{
if (fclose(full_backup->hdr_map.fp))
elog(ERROR, "Cannot close file \"%s\"", full_backup->hdr_map.path);
/* sync new header map to dist */
if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Cannot sync temp header map \"%s\": %s",
full_backup->hdr_map.path_tmp, strerror(errno));
/* Replace old header map with new one */
if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path) == -1)
elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s",
full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno));
}
/*
* Update FULL backup metadata.
* We cannot set backup status to OK just yet,
@@ -847,6 +864,12 @@ merge_rename:
full_backup->root_dir = pgut_strdup(destination_path);
}
/* Reinit some path variables */
join_path_components(full_backup->database_dir, full_backup->root_dir, DATABASE_DIR);
join_path_components(full_backup->hdr_map.path, full_backup->database_dir, HEADER_MAP);
join_path_components(full_backup->hdr_map.path_tmp, full_backup->database_dir, HEADER_MAP_TMP);
full_backup->hdr_map.fp = NULL;
/* If we crash here, it will produce full backup in MERGED
* status, located in directory with wrong backup id.
* It should not be a problem.
@@ -1033,6 +1056,7 @@ merge_files(void *arg)
if (file &&
file->n_blocks == dest_file->n_blocks)
{
BackupPageHeader2 *headers = NULL;
elog(VERBOSE, "The file didn`t changed since FULL backup, skip merge: \"%s\"",
file->rel_path);
@@ -1052,6 +1076,18 @@ merge_files(void *arg)
else
tmp_file->uncompressed_size = tmp_file->write_size;
/* Copy header metadata from old map into a new one */
tmp_file->n_headers = file->n_headers;
headers = get_data_file_headers(&(arguments->full_backup->hdr_map), file,
parse_program_version(arguments->full_backup->program_version));
/* sanity */
if (!headers && file->n_headers > 0)
elog(ERROR, "Failed to get headers for file \"%s\"", file->rel_path);
write_page_headers(headers, tmp_file, &(arguments->full_backup->hdr_map), true);
pg_free(headers);
//TODO: report in_place merge bytes.
goto done;
}
@@ -1205,16 +1241,14 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
* 16KB.
* TODO: maybe we should just trust dest_file->n_blocks?
* No, we can`t, because current binary can be used to merge
* 2 backups of old versions, were n_blocks is missing.
* 2 backups of old versions, where n_blocks is missing.
*/
backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2,
InvalidXLogRecPtr, BACKUP_MODE_FULL,
dest_backup->compress_alg, dest_backup->compress_level,
dest_backup->checksum_version, 0, NULL,
/* TODO: add header map */
NULL, false);
&(full_backup->hdr_map), true);
/* drop restored temp file */
if (unlink(to_fullpath_tmp1) == -1)
src/pg_probackup.h
+5
View File
@@ -67,6 +67,7 @@ extern const char *PROGRAM_EMAIL;
#define EXTERNAL_DIR "external_directories/externaldir"
#define DATABASE_MAP "database_map"
#define HEADER_MAP "block_header_map"
#define HEADER_MAP_TMP "block_header_map_tmp"
/* Timeout defaults */
#define ARCHIVE_TIMEOUT_DEFAULT 300
@@ -363,6 +364,7 @@ typedef struct PGNodeInfo
typedef struct HeaderMap
{
char *path;
char *path_tmp; /* used only in merge */
FILE *fp;
off_t offset;
pthread_mutex_t mutex;
@@ -1014,6 +1016,9 @@ extern pid_t check_postmaster(const char *pgdata);
extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map);
extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version);
extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge);
/* parsexlog.c */
extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size,
XLogRecPtr startpoint, TimeLineID start_tli,