db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-09 14:45:47 +02:00
Code Issues Releases Activity

[Issue #228]: added page header map compression

Browse Source
This commit is contained in:
Grigory Smolkin 2020-06-20 19:14:38 +03:00
parent a0d31d530b
commit 09e87a54f7
4 changed files with 67 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/catalog.c
View File

@ -1926,6 +1926,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers);
len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc);
len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off);
len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size);
}
sprintf(line+len, "}\n");

67
src/data.c
View File

@ -2130,6 +2130,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
FILE *in = NULL;
pg_crc32 hdr_crc;
BackupPageHeader2 *headers = NULL;
/* header decompression */
char *zheaders = NULL;
const char *errormsg = NULL;
if (backup_version < 20400)
return NULL;
@ -2150,14 +2153,33 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
file->hdr_off, hdr_map->path, strerror(errno));
/*
* the actual number of headers in header file is n+1, last one is a dummy header,
* used for calculation of compressed_size for actual last header.
* The actual number of headers in header file is n+1, last one is a dummy header,
* used for calculation of read_len for actual last header.
*/
read_len = (file->n_headers+1) * sizeof(BackupPageHeader2);
headers = pgut_malloc(read_len);
if (fread(headers, 1, read_len, in) != read_len)
elog(ERROR, "Cannot read header file \"%s\": %s", hdr_map->path, strerror(errno));
/* allocate memory for compressed and uncompressed headers */
headers = pgut_malloc(read_len);
memset(headers, 0, read_len);
zheaders = pgut_malloc(file->hdr_size);
memset(zheaders, 0, file->hdr_size);
if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size)
elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s",
file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno));
// elog(INFO, "zsize: %i, size: %i", file->hdr_size, read_len);
if (do_decompress(headers, read_len, zheaders, file->hdr_size,
ZLIB_COMPRESS, &errormsg) != read_len)
{
if (errormsg)
elog(ERROR, "An error occured during metadata decompression for file \"%s\": %s",
file->rel_path, errormsg);
else
elog(ERROR, "An error occured during metadata decompression for file \"%s\"",
file->rel_path);
}
/* validate checksum */
INIT_FILE_CRC32(true, hdr_crc);
@ -2171,6 +2193,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version)
if (fclose(in))
elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno));
pg_free(zheaders);
return headers;
}
@ -2179,6 +2203,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
{
size_t read_len = 0;
char *map_path = NULL;
/* header compression */
int z_len = 0;
char *zheaders = NULL;
const char *errormsg = NULL;
if (file->n_headers <= 0)
return;
@ -2219,11 +2247,34 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len);
FIN_FILE_CRC32(true, file->hdr_crc);
if (fwrite(headers, 1, read_len, hdr_map->fp) != read_len)
zheaders = pgut_malloc(read_len*2);
memset(zheaders, 0, read_len*2);
z_len = do_compress(zheaders, read_len*2, headers,
read_len, ZLIB_COMPRESS, 1, &errormsg);
if (z_len < 0)
{
if (errormsg)
elog(ERROR, "An error occured during compressing metadata for file \"%s\": %s",
file->rel_path, errormsg);
else
elog(ERROR, "An error occured during compressing metadata for file \"%s\": %i",
file->rel_path, z_len);
}
if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len)
elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno));
elog(VERBOSE, "Writing header map for file \"%s\" offset: %lu, len: %lu, crc: %u",
file->rel_path, file->hdr_off, read_len, file->hdr_crc);
elog(VERBOSE, "Writing header map for file \"%s\" offset: %li, len: %i, crc: %u",
file->rel_path, file->hdr_off, z_len, file->hdr_crc);
// elog(INFO, "File: %s, Unzip: %i, zip: %i", file->rel_path, read_len, z_len);
file->hdr_size = z_len;
/* End critical section */
pthread_mutex_unlock(&(hdr_map->mutex));
pg_free(zheaders);
}

6
src/dir.c
View File

@ -1552,7 +1552,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
n_headers,
dbOid, /* used for partial restore */
hdr_crc,
hdr_off;
hdr_off,
hdr_size;
pgFile *file;
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
@ -1602,6 +1603,9 @@ dir_read_file_list(const char *root, const char *external_prefix,
if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false))
file->hdr_off = hdr_off;
if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false))
file->hdr_size = (int) hdr_size;
parray_append(files, file);
}

3
src/pg_probackup.h
View File

@ -214,10 +214,11 @@ typedef struct pgFile
may take up to 16kB per file */
bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap,
* i.e. datafiles without _ptrack */
/* coordinates in header map */
/* Coordinates in header map */
int n_headers; /* number of blocks in the data file in backup */
pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */
off_t hdr_off; /* offset in header map */
int hdr_size; /* offset in header map */
} pgFile;
typedef struct page_map_entry