db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-09 14:45:47 +02:00
Code Issues Releases Activity

[Issue #228]: fixes

Browse Source
This commit is contained in:
Grigory Smolkin 2020-06-21 03:17:28 +03:00
parent e49a490943
commit ac1ff37ada
4 changed files with 62 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/backup.c
View File

@ -599,7 +599,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
} }
/* close and sync page header map */ /* close and sync page header map */
if (current.hdr_map.w_fp) if (current.hdr_map.fp)
{ {
cleanup_header_map(&(current.hdr_map)); cleanup_header_map(&(current.hdr_map));

134
src/data.c
View File

@ -2126,6 +2126,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f
BackupPageHeader2* BackupPageHeader2*
get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
{ {
FILE *in = NULL;
size_t read_len = 0; size_t read_len = 0;
pg_crc32 hdr_crc; pg_crc32 hdr_crc;
BackupPageHeader2 *headers = NULL; BackupPageHeader2 *headers = NULL;
@ -2140,35 +2141,15 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
if (file->n_headers <= 0) if (file->n_headers <= 0)
return NULL; return NULL;
// in = fopen(hdr_map->path, PG_BINARY_R); /* TODO: consider to make this descriptor thread-specific */
// in = fopen(hdr_map->path, PG_BINARY_R);
// if (!in)
// elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno));
if (!hdr_map->r_fp) if (!in)
{ elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno));
pthread_lock(&(hdr_map->mutex)); /* disable buffering for header file */
setvbuf(in, NULL, _IONBF, BUFSIZ);
/* it is possible for another contender got here first, so double check */ if (fseek(in, file->hdr_off, SEEK_SET))
if (!hdr_map->r_fp) /* this file will be closed in restore.c and merge.c */
{
elog(LOG, "Opening page header map \"%s\"", hdr_map->path);
hdr_map->r_fp = fopen(hdr_map->path, PG_BINARY_R);
if (hdr_map->r_fp == NULL)
elog(ERROR, "Cannot open header file \"%s\": %s",
hdr_map->path, strerror(errno));
/* enable buffering for header file */
hdr_map->r_buf = pgut_malloc(LARGE_CHUNK_SIZE);
setvbuf(hdr_map->r_fp, hdr_map->r_buf, _IOFBF, LARGE_CHUNK_SIZE);
}
/* End critical section */
pthread_mutex_unlock(&(hdr_map->mutex));
}
if (fseek(hdr_map->r_fp, file->hdr_off, SEEK_SET))
elog(ERROR, "Cannot seek to position %lu in page header map \"%s\": %s", elog(ERROR, "Cannot seek to position %lu in page header map \"%s\": %s",
file->hdr_off, hdr_map->path, strerror(errno)); file->hdr_off, hdr_map->path, strerror(errno));
@ -2184,7 +2165,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
zheaders = pgut_malloc(file->hdr_size); zheaders = pgut_malloc(file->hdr_size);
memset(zheaders, 0, file->hdr_size); memset(zheaders, 0, file->hdr_size);
if (fread(zheaders, 1, file->hdr_size, hdr_map->r_fp) != file->hdr_size) if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size)
elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s", elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s",
file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno));
@ -2211,6 +2192,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u", elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u",
file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc);
if (fclose(in))
elog(ERROR, "Cannot close file \"%s\"", hdr_map->path);
pg_free(zheaders); pg_free(zheaders);
return headers; return headers;
@ -2231,33 +2215,6 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
/* when running merge we must save headers into the temp map */ /* when running merge we must save headers into the temp map */
map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path;
/* writing to header map must be serialized */
pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */
if (!hdr_map->w_fp)
{
elog(LOG, "Creating page header map \"%s\"", map_path);
hdr_map->w_fp = fopen(map_path, PG_BINARY_W);
if (hdr_map->w_fp == NULL)
elog(ERROR, "Cannot open header file \"%s\": %s",
map_path, strerror(errno));
/* enable buffering for header file */
hdr_map->w_buf = pgut_malloc(LARGE_CHUNK_SIZE);
setvbuf(hdr_map->w_fp, hdr_map->w_buf, _IOFBF, LARGE_CHUNK_SIZE);
/* update file permission */
if (chmod(map_path, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", map_path,
strerror(errno));
file->hdr_off = 0;
}
else
file->hdr_off = ftell(hdr_map->w_fp); /* TODO: replace by counter */
read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); read_len = (file->n_headers+1) * sizeof(BackupPageHeader2);
/* calculate checksums */ /* calculate checksums */
@ -2268,9 +2225,36 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
zheaders = pgut_malloc(read_len*2); zheaders = pgut_malloc(read_len*2);
memset(zheaders, 0, read_len*2); memset(zheaders, 0, read_len*2);
/* compress headers */
z_len = do_compress(zheaders, read_len*2, headers, z_len = do_compress(zheaders, read_len*2, headers,
read_len, ZLIB_COMPRESS, 1, &errormsg); read_len, ZLIB_COMPRESS, 1, &errormsg);
/* writing to header map must be serialized */
pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */
if (!hdr_map->fp)
{
elog(LOG, "Creating page header map \"%s\"", map_path);
hdr_map->fp = fopen(map_path, PG_BINARY_W);
if (hdr_map->fp == NULL)
elog(ERROR, "Cannot open header file \"%s\": %s",
map_path, strerror(errno));
/* enable buffering for header file */
hdr_map->buf = pgut_malloc(LARGE_CHUNK_SIZE);
setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, LARGE_CHUNK_SIZE);
/* update file permission */
if (chmod(map_path, FILE_PERMISSION) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", map_path,
strerror(errno));
file->hdr_off = 0;
}
else
file->hdr_off = hdr_map->offset;
if (z_len <= 0) if (z_len <= 0)
{ {
if (errormsg) if (errormsg)
@ -2281,15 +2265,14 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
file->rel_path, z_len); file->rel_path, z_len);
} }
if (fwrite(zheaders, 1, z_len, hdr_map->w_fp) != z_len) elog(VERBOSE, "Writing headers for file \"%s\" offset: %li, len: %i, crc: %u",
elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno));
elog(VERBOSE, "Writing header map for file \"%s\" offset: %li, len: %i, crc: %u",
file->rel_path, file->hdr_off, z_len, file->hdr_crc); file->rel_path, file->hdr_off, z_len, file->hdr_crc);
elog(INFO, "File: %s, Unzip: %li, zip: %i", file->rel_path, read_len, z_len); if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len)
elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno));
file->hdr_size = z_len; file->hdr_size = z_len; /* save the length of compressed headers */
hdr_map->offset += z_len; /* update current offset in map */
/* End critical section */ /* End critical section */
pthread_mutex_unlock(&(hdr_map->mutex)); pthread_mutex_unlock(&(hdr_map->mutex));
@ -2300,10 +2283,8 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
void void
init_header_map(pgBackup *backup) init_header_map(pgBackup *backup)
{ {
backup->hdr_map.r_fp = NULL; backup->hdr_map.fp = NULL;
backup->hdr_map.w_fp = NULL; backup->hdr_map.buf = NULL;
backup->hdr_map.r_buf = NULL;
backup->hdr_map.w_buf = NULL;
join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP);
join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP);
backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
@ -2312,22 +2293,11 @@ init_header_map(pgBackup *backup)
void void
cleanup_header_map(HeaderMap *hdr_map) cleanup_header_map(HeaderMap *hdr_map)
{ {
/* cleanup descriptor */
/* cleanup read descriptor */ if (hdr_map->fp && fclose(hdr_map->fp))
if (hdr_map->r_fp && fclose(hdr_map->r_fp))
elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); elog(ERROR, "Cannot close file \"%s\"", hdr_map->path);
hdr_map->fp = NULL;
hdr_map->r_fp = NULL; hdr_map->offset = 0;
pg_free(hdr_map->r_buf); pg_free(hdr_map->buf);
hdr_map->r_buf = NULL; hdr_map->buf = NULL;
hdr_map->r_offset = 0;
/* cleanup write descriptor */
if (hdr_map->w_fp && fclose(hdr_map->w_fp))
elog(ERROR, "Cannot close file \"%s\"", hdr_map->path);
hdr_map->w_fp = NULL;
pg_free(hdr_map->w_buf);
hdr_map->w_buf = NULL;
hdr_map->w_offset = 0;
} }

6
src/merge.c
View File

@ -705,12 +705,12 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
elog(ERROR, "Backup files merging failed, time elapsed: %s", elog(ERROR, "Backup files merging failed, time elapsed: %s",
pretty_time); pretty_time);
/* If temp header map descriptor is open, then close it and make rename */ /* If temp header map is open, then close it and make rename */
if (full_backup->hdr_map.w_fp) if (full_backup->hdr_map.fp)
{ {
cleanup_header_map(&(full_backup->hdr_map)); cleanup_header_map(&(full_backup->hdr_map));
/* sync new header map to dist */ /* sync new header map to disk */
if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Cannot sync temp header map \"%s\": %s", elog(ERROR, "Cannot sync temp header map \"%s\": %s",
full_backup->hdr_map.path_tmp, strerror(errno)); full_backup->hdr_map.path_tmp, strerror(errno));

15
src/pg_probackup.h
View File

@ -66,8 +66,8 @@ extern const char *PROGRAM_EMAIL;
#define PG_TABLESPACE_MAP_FILE "tablespace_map" #define PG_TABLESPACE_MAP_FILE "tablespace_map"
#define EXTERNAL_DIR "external_directories/externaldir" #define EXTERNAL_DIR "external_directories/externaldir"
#define DATABASE_MAP "database_map" #define DATABASE_MAP "database_map"
#define HEADER_MAP "block_header_map" #define HEADER_MAP "page_header_map"
#define HEADER_MAP_TMP "block_header_map_tmp" #define HEADER_MAP_TMP "page_header_map_tmp"
/* Timeout defaults */ /* Timeout defaults */
#define ARCHIVE_TIMEOUT_DEFAULT 300 #define ARCHIVE_TIMEOUT_DEFAULT 300
@ -366,13 +366,10 @@ typedef struct PGNodeInfo
typedef struct HeaderMap typedef struct HeaderMap
{ {
char path[MAXPGPATH]; char path[MAXPGPATH];
char path_tmp[MAXPGPATH]; /* used only in merge */ char path_tmp[MAXPGPATH]; /* used only in merge */
char *r_buf; /* buffer */ FILE *fp; /* used only for writing */
char *w_buf; /* buffer */ char *buf; /* buffer */
FILE *r_fp; /* descriptor used for reading */ off_t offset; /* current position in fp */
FILE *w_fp; /* descriptor used for writing */
off_t r_offset; /* current position in r_fp */
off_t w_offset; /* current position in w_fp */
pthread_mutex_t mutex; pthread_mutex_t mutex;
} HeaderMap; } HeaderMap;