db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-10 23:29:34 +02:00
Code Issues Releases Activity

Merge branch 'issue_201' into release_2_4

Browse Source
This commit is contained in:
Grigory Smolkin 2020-05-25 14:44:23 +03:00
commit f2d2bab6d7
8 changed files with 175 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/backup.c
View File

@ -2225,7 +2225,7 @@ parse_filelist_filenames(parray *files, const char *root)
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
file->name && file->name[0])
{
if (file->forkName == INIT)
if (file->forkName == init)
{
/*
* Do not backup files of unlogged relations.

71
src/data.c
View File

@ -861,13 +861,25 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
* Apply changed blocks to destination file from every backup in parent chain.
*/
size_t
restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath)
restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
const char *to_fullpath, bool use_bitmap)
{
int i;
size_t total_write_len = 0;
char *in_buf = pgut_malloc(STDIO_BUFSIZE);
int backup_seq = 0;
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
// FULL -> INCR -> DEST
// 2 1 0
if (use_bitmap)
/* start with dest backup */
backup_seq = 0;
else
/* start with full backup */
backup_seq = parray_num(parent_chain) - 1;
// for (i = parray_num(parent_chain) - 1; i >= 0; i--)
// for (i = 0; i < parray_num(parent_chain); i++)
while (backup_seq >= 0 && backup_seq < parray_num(parent_chain))
{
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
@ -876,7 +888,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
pgFile **res_file = NULL;
pgFile *tmp_file = NULL;
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq);
if (use_bitmap)
backup_seq++;
else
backup_seq--;
/* lookup file in intermediate backup */
res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal);
@ -923,20 +940,29 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
*/
total_write_len += restore_data_file_internal(in, out, tmp_file,
parse_program_version(backup->program_version),
from_fullpath, to_fullpath, dest_file->n_blocks);
from_fullpath, to_fullpath, dest_file->n_blocks,
use_bitmap ? &(dest_file)->pagemap : NULL);
if (fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
strerror(errno));
// datapagemap_print_debug(&(dest_file)->pagemap);
}
pg_free(in_buf);
return total_write_len;
}
/* Restore block from "in" file to "out" file.
* If "nblocks" is greater than zero, then skip restoring blocks,
* whose position if greater than "nblocks".
* If map is NULL, then page bitmap cannot be used for restore optimization
*/
size_t
restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
const char *from_fullpath, const char *to_fullpath, int nblocks)
const char *from_fullpath, const char *to_fullpath, int nblocks,
datapagemap_t *map)
{
BackupPageHeader header;
BlockNumber blknum = 0;
@ -994,9 +1020,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
}
/* sanity? */
if (header.block < blknum)
elog(ERROR, "Backup is broken at block %u of \"%s\"",
blknum, from_fullpath);
// if (header.block < blknum)
// elog(ERROR, "Backup is broken at block %u of \"%s\"",
// blknum, from_fullpath);
blknum = header.block;
@ -1044,6 +1070,16 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
if (compressed_size > BLCKSZ)
elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum);
/* if this page is marked as already restored, then skip it */
if (map && datapagemap_is_set(map, blknum))
{
/* skip to the next page */
if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0)
elog(ERROR, "Cannot seek block %u of '%s': %s",
blknum, from_fullpath, strerror(errno));
continue;
}
/* read a page from file */
read_len = fread(page.data, 1, MAXALIGN(compressed_size), in);
@ -1099,6 +1135,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
write_len += BLCKSZ;
cur_pos = write_pos + BLCKSZ; /* update current write position */
if (map)
datapagemap_add(map, blknum);
}
elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len);
@ -1152,7 +1191,7 @@ size_t
restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath)
{
int i;
// int i;
char from_root[MAXPGPATH];
char from_fullpath[MAXPGPATH];
FILE *in = NULL;
@ -1175,12 +1214,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
* Full copy is latest possible destination file with size equal or
* greater than zero.
*/
for (i = 1; i < parray_num(parent_chain); i++)
tmp_backup = dest_backup->parent_backup_link;
while (tmp_backup)
{
pgFile **res_file = NULL;
tmp_backup = (pgBackup *) parray_get(parent_chain, i);
/* lookup file in intermediate backup */
res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal);
tmp_file = (res_file) ? *res_file : NULL;
@ -1203,6 +1241,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
/* Full copy is found */
if (tmp_file->write_size > 0)
break;
tmp_backup = tmp_backup->parent_backup_link;
}
}
@ -1214,6 +1254,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
if (!tmp_file)
elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath);
if (tmp_file->write_size <= 0)
elog(ERROR, "Full copy of non-data file has invalid size. "
"Metadata corruption in backup %s in file: \"%s\"",
base36enc(tmp_backup->start_time), to_fullpath);
if (tmp_file->external_dir_num == 0)
join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR);
else

16
src/dir.c
View File

@ -720,22 +720,22 @@ dir_check_file(pgFile *file)
{
/* Auxiliary fork of the relfile */
if (strcmp(fork_name, "vm") == 0)
file->forkName = VM;
file->forkName = vm;
else if (strcmp(fork_name, "fsm") == 0)
file->forkName = FSM;
file->forkName = fsm;
else if (strcmp(fork_name, "cfm") == 0)
file->forkName = CFM;
else if (strcmp(fork_name, "init") == 0)
file->forkName = INIT;
file->forkName = cfm;
else if (strcmp(fork_name, "ptrack") == 0)
file->forkName = PTRACK;
file->forkName = ptrack;
else if (strcmp(fork_name, "init") == 0)
file->forkName = init;
/* Do not backup ptrack files */
if (file->forkName == PTRACK)
if (file->forkName == ptrack)
return CHECK_FALSE;
}
else

15
src/merge.c
View File

@ -28,6 +28,7 @@ typedef struct
// size_t in_place_merge_bytes;
bool compression_match;
bool program_version_match;
bool use_bitmap;
/*
* Return value from the thread.
@ -47,7 +48,7 @@ get_external_index(const char *key, const parray *list);
static void
merge_data_file(parray *parent_chain, pgBackup *full_backup,
pgBackup *dest_backup, pgFile *dest_file,
pgFile *tmp_file, const char *to_root);
pgFile *tmp_file, const char *to_root, bool use_bitmap);
static void
merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
@ -439,6 +440,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
*dest_externals = NULL;
parray *result_filelist = NULL;
bool use_bitmap = true;
// size_t total_in_place_merge_bytes = 0;
pthread_t *threads = NULL;
@ -601,6 +603,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
if (full_externals && dest_externals)
reorder_external_dirs(full_backup, full_externals, dest_externals);
if (parse_program_version(dest_backup->program_version) < 20300)
use_bitmap = false;
/* Setup threads */
for (i = 0; i < parray_num(dest_backup->files); i++)
{
@ -639,6 +644,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
arg->compression_match = compression_match;
arg->program_version_match = program_version_match;
arg->use_bitmap = use_bitmap;
/* By default there are some error */
arg->ret = 1;
@ -1028,7 +1034,8 @@ merge_files(void *arg)
arguments->full_backup,
arguments->dest_backup,
dest_file, tmp_file,
arguments->full_database_dir);
arguments->full_database_dir,
arguments->use_bitmap);
else
merge_non_data_file(arguments->parent_chain,
arguments->full_backup,
@ -1128,7 +1135,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external,
void
merge_data_file(parray *parent_chain, pgBackup *full_backup,
pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file,
const char *full_database_dir)
const char *full_database_dir, bool use_bitmap)
{
FILE *out = NULL;
char *buffer = pgut_malloc(STDIO_BUFSIZE);
@ -1154,7 +1161,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE);
/* restore file into temp file */
tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1);
tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap);
fclose(out);
pg_free(buffer);

21
src/pg_probackup.h
View File

@ -124,11 +124,11 @@ typedef enum CompressAlg
typedef enum ForkName
{
VM,
FSM,
CFM,
INIT,
PTRACK
vm,
fsm,
cfm,
init,
ptrack
} ForkName;
#define INIT_FILE_CRC32(use_crc32c, crc) \
@ -935,9 +935,10 @@ extern void backup_non_data_file_internal(const char *from_fullpath,
bool missing_ok);
extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file,
FILE *out, const char *to_fullpath);
FILE *out, const char *to_fullpath, bool use_bitmap);
extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
const char *from_fullpath, const char *to_fullpath, int nblocks);
const char *from_fullpath, const char *to_fullpath, int nblocks,
datapagemap_t *map);
extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath);
extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
@ -1056,4 +1057,10 @@ extern void get_header_errormsg(Page page, char **errormsg);
extern void get_checksum_errormsg(Page page, char **errormsg,
BlockNumber absolute_blkno);
extern bool
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno);
extern void
datapagemap_print_debug(datapagemap_t *map);
#endif /* PG_PROBACKUP_H */

23
src/restore.c
View File

@ -27,6 +27,7 @@ typedef struct
bool skip_external_dirs;
const char *to_root;
size_t restored_bytes;
bool use_bitmap;
/*
* Return value from the thread.
@ -501,6 +502,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
pthread_t *threads;
restore_files_arg *threads_args;
bool restore_isok = true;
bool use_bitmap = true;
/* fancy reporting */
char pretty_dest_bytes[20];
@ -560,6 +562,17 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
}
/* If dest backup version is older than 2.3.0, then bitmap optimization
* is impossible to use, because bitmap restore rely on pgFile.n_blocks,
* which is not always available in old backups.
*/
if (parse_program_version(dest_backup->program_version) < 20300)
use_bitmap = false;
/* There is no point in bitmap restore, when restoring a single FULL backup */
if (parray_num(parent_chain) == 1)
use_bitmap = false;
/*
* Restore dest_backup internal directories.
*/
@ -645,6 +658,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
arg->dbOid_exclude_list = dbOid_exclude_list;
arg->skip_external_dirs = params->skip_external_dirs;
arg->to_root = pgdata_path;
arg->use_bitmap = use_bitmap;
threads_args[i].restored_bytes = 0;
/* By default there are some error */
threads_args[i].ret = 1;
@ -859,7 +873,8 @@ restore_files(void *arg)
setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE);
/* Destination file is data file */
arguments->restored_bytes += restore_data_file(arguments->parent_chain,
dest_file, out, to_fullpath);
dest_file, out, to_fullpath,
arguments->use_bitmap);
}
else
{
@ -871,6 +886,9 @@ restore_files(void *arg)
arguments->dest_backup, dest_file, out, to_fullpath);
}
/* free pagemap used for restore optimization */
pg_free(dest_file->pagemap.bitmap);
done:
/* close file */
if (fio_fclose(out) != 0)
@ -973,6 +991,9 @@ create_recovery_conf(time_t backup_id,
elog(ERROR, "cannot open file \"%s\": %s", path,
strerror(errno));
if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno));
#if PG_VERSION_NUM >= 120000
fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n",
PROGRAM_VERSION);

54
src/util.c
View File

@ -495,3 +495,57 @@ str2status(const char *status)
return BACKUP_STATUS_INVALID;
}
bool
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno)
{
int offset;
int bitno;
offset = blkno / 8;
bitno = blkno % 8;
/* enlarge or create bitmap if needed */
if (map->bitmapsize <= offset)
{
int oldsize = map->bitmapsize;
int newsize;
/*
* The minimum to hold the new bit is offset + 1. But add some
* headroom, so that we don't need to repeatedly enlarge the bitmap in
* the common case that blocks are modified in order, from beginning
* of a relation to the end.
*/
newsize = offset + 1;
newsize += 10;
map->bitmap = pg_realloc(map->bitmap, newsize);
/* zero out the newly allocated region */
memset(&map->bitmap[oldsize], 0, newsize - oldsize);
map->bitmapsize = newsize;
}
//datapagemap_print(map);
/* check the bit */
return map->bitmap[offset] & (1 << bitno);
}
/*
* A debugging aid. Prints out the contents of the page map.
*/
void
datapagemap_print_debug(datapagemap_t *map)
{
datapagemap_iterator_t *iter;
BlockNumber blocknum;
iter = datapagemap_iterate(map);
while (datapagemap_next(iter, &blocknum))
elog(INFO, " block %u", blocknum);
pg_free(iter);
}

14
tests/retention.py
View File

@ -629,15 +629,15 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.pgbench_init(scale=3)
node.pgbench_init(scale=5)
# Take FULL BACKUPs
backup_id_a = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
backup_id_b = self.backup_node(backup_dir, 'node', node)
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
# Change FULL B backup status to ERROR
@ -648,7 +648,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
pgdata_a1 = self.pgdata_content(node.data_dir)
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
# PAGEa1 OK
@ -666,20 +666,20 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
page_id_b1 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
page_id_b2 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
page_id_b3 = self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgdata_b3 = self.pgdata_content(node.data_dir)
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
pgbench.wait()
# PAGEb3 OK