mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2025-01-10 23:29:34 +02:00
Merge branch 'issue_201' into release_2_4
This commit is contained in:
commit
f2d2bab6d7
@ -2225,7 +2225,7 @@ parse_filelist_filenames(parray *files, const char *root)
|
||||
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
|
||||
file->name && file->name[0])
|
||||
{
|
||||
if (file->forkName == INIT)
|
||||
if (file->forkName == init)
|
||||
{
|
||||
/*
|
||||
* Do not backup files of unlogged relations.
|
||||
|
71
src/data.c
71
src/data.c
@ -861,13 +861,25 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
|
||||
* Apply changed blocks to destination file from every backup in parent chain.
|
||||
*/
|
||||
size_t
|
||||
restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath)
|
||||
restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
|
||||
const char *to_fullpath, bool use_bitmap)
|
||||
{
|
||||
int i;
|
||||
size_t total_write_len = 0;
|
||||
char *in_buf = pgut_malloc(STDIO_BUFSIZE);
|
||||
int backup_seq = 0;
|
||||
|
||||
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||
// FULL -> INCR -> DEST
|
||||
// 2 1 0
|
||||
if (use_bitmap)
|
||||
/* start with dest backup */
|
||||
backup_seq = 0;
|
||||
else
|
||||
/* start with full backup */
|
||||
backup_seq = parray_num(parent_chain) - 1;
|
||||
|
||||
// for (i = parray_num(parent_chain) - 1; i >= 0; i--)
|
||||
// for (i = 0; i < parray_num(parent_chain); i++)
|
||||
while (backup_seq >= 0 && backup_seq < parray_num(parent_chain))
|
||||
{
|
||||
char from_root[MAXPGPATH];
|
||||
char from_fullpath[MAXPGPATH];
|
||||
@ -876,7 +888,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
|
||||
pgFile **res_file = NULL;
|
||||
pgFile *tmp_file = NULL;
|
||||
|
||||
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
|
||||
pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq);
|
||||
|
||||
if (use_bitmap)
|
||||
backup_seq++;
|
||||
else
|
||||
backup_seq--;
|
||||
|
||||
/* lookup file in intermediate backup */
|
||||
res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal);
|
||||
@ -923,20 +940,29 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
|
||||
*/
|
||||
total_write_len += restore_data_file_internal(in, out, tmp_file,
|
||||
parse_program_version(backup->program_version),
|
||||
from_fullpath, to_fullpath, dest_file->n_blocks);
|
||||
from_fullpath, to_fullpath, dest_file->n_blocks,
|
||||
use_bitmap ? &(dest_file)->pagemap : NULL);
|
||||
|
||||
if (fclose(in) != 0)
|
||||
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
|
||||
strerror(errno));
|
||||
|
||||
// datapagemap_print_debug(&(dest_file)->pagemap);
|
||||
}
|
||||
pg_free(in_buf);
|
||||
|
||||
return total_write_len;
|
||||
}
|
||||
|
||||
/* Restore block from "in" file to "out" file.
|
||||
* If "nblocks" is greater than zero, then skip restoring blocks,
|
||||
* whose position if greater than "nblocks".
|
||||
* If map is NULL, then page bitmap cannot be used for restore optimization
|
||||
*/
|
||||
size_t
|
||||
restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
|
||||
const char *from_fullpath, const char *to_fullpath, int nblocks)
|
||||
const char *from_fullpath, const char *to_fullpath, int nblocks,
|
||||
datapagemap_t *map)
|
||||
{
|
||||
BackupPageHeader header;
|
||||
BlockNumber blknum = 0;
|
||||
@ -994,9 +1020,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
|
||||
}
|
||||
|
||||
/* sanity? */
|
||||
if (header.block < blknum)
|
||||
elog(ERROR, "Backup is broken at block %u of \"%s\"",
|
||||
blknum, from_fullpath);
|
||||
// if (header.block < blknum)
|
||||
// elog(ERROR, "Backup is broken at block %u of \"%s\"",
|
||||
// blknum, from_fullpath);
|
||||
|
||||
blknum = header.block;
|
||||
|
||||
@ -1044,6 +1070,16 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
|
||||
if (compressed_size > BLCKSZ)
|
||||
elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum);
|
||||
|
||||
/* if this page is marked as already restored, then skip it */
|
||||
if (map && datapagemap_is_set(map, blknum))
|
||||
{
|
||||
/* skip to the next page */
|
||||
if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0)
|
||||
elog(ERROR, "Cannot seek block %u of '%s': %s",
|
||||
blknum, from_fullpath, strerror(errno));
|
||||
continue;
|
||||
}
|
||||
|
||||
/* read a page from file */
|
||||
read_len = fread(page.data, 1, MAXALIGN(compressed_size), in);
|
||||
|
||||
@ -1099,6 +1135,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
|
||||
|
||||
write_len += BLCKSZ;
|
||||
cur_pos = write_pos + BLCKSZ; /* update current write position */
|
||||
|
||||
if (map)
|
||||
datapagemap_add(map, blknum);
|
||||
}
|
||||
|
||||
elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len);
|
||||
@ -1152,7 +1191,7 @@ size_t
|
||||
restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
|
||||
pgFile *dest_file, FILE *out, const char *to_fullpath)
|
||||
{
|
||||
int i;
|
||||
// int i;
|
||||
char from_root[MAXPGPATH];
|
||||
char from_fullpath[MAXPGPATH];
|
||||
FILE *in = NULL;
|
||||
@ -1175,12 +1214,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
|
||||
* Full copy is latest possible destination file with size equal or
|
||||
* greater than zero.
|
||||
*/
|
||||
for (i = 1; i < parray_num(parent_chain); i++)
|
||||
tmp_backup = dest_backup->parent_backup_link;
|
||||
while (tmp_backup)
|
||||
{
|
||||
pgFile **res_file = NULL;
|
||||
|
||||
tmp_backup = (pgBackup *) parray_get(parent_chain, i);
|
||||
|
||||
/* lookup file in intermediate backup */
|
||||
res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal);
|
||||
tmp_file = (res_file) ? *res_file : NULL;
|
||||
@ -1203,6 +1241,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
|
||||
/* Full copy is found */
|
||||
if (tmp_file->write_size > 0)
|
||||
break;
|
||||
|
||||
tmp_backup = tmp_backup->parent_backup_link;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1214,6 +1254,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
|
||||
if (!tmp_file)
|
||||
elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath);
|
||||
|
||||
if (tmp_file->write_size <= 0)
|
||||
elog(ERROR, "Full copy of non-data file has invalid size. "
|
||||
"Metadata corruption in backup %s in file: \"%s\"",
|
||||
base36enc(tmp_backup->start_time), to_fullpath);
|
||||
|
||||
if (tmp_file->external_dir_num == 0)
|
||||
join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR);
|
||||
else
|
||||
|
16
src/dir.c
16
src/dir.c
@ -720,22 +720,22 @@ dir_check_file(pgFile *file)
|
||||
{
|
||||
/* Auxiliary fork of the relfile */
|
||||
if (strcmp(fork_name, "vm") == 0)
|
||||
file->forkName = VM;
|
||||
file->forkName = vm;
|
||||
|
||||
else if (strcmp(fork_name, "fsm") == 0)
|
||||
file->forkName = FSM;
|
||||
file->forkName = fsm;
|
||||
|
||||
else if (strcmp(fork_name, "cfm") == 0)
|
||||
file->forkName = CFM;
|
||||
|
||||
else if (strcmp(fork_name, "init") == 0)
|
||||
file->forkName = INIT;
|
||||
file->forkName = cfm;
|
||||
|
||||
else if (strcmp(fork_name, "ptrack") == 0)
|
||||
file->forkName = PTRACK;
|
||||
file->forkName = ptrack;
|
||||
|
||||
else if (strcmp(fork_name, "init") == 0)
|
||||
file->forkName = init;
|
||||
|
||||
/* Do not backup ptrack files */
|
||||
if (file->forkName == PTRACK)
|
||||
if (file->forkName == ptrack)
|
||||
return CHECK_FALSE;
|
||||
}
|
||||
else
|
||||
|
15
src/merge.c
15
src/merge.c
@ -28,6 +28,7 @@ typedef struct
|
||||
// size_t in_place_merge_bytes;
|
||||
bool compression_match;
|
||||
bool program_version_match;
|
||||
bool use_bitmap;
|
||||
|
||||
/*
|
||||
* Return value from the thread.
|
||||
@ -47,7 +48,7 @@ get_external_index(const char *key, const parray *list);
|
||||
static void
|
||||
merge_data_file(parray *parent_chain, pgBackup *full_backup,
|
||||
pgBackup *dest_backup, pgFile *dest_file,
|
||||
pgFile *tmp_file, const char *to_root);
|
||||
pgFile *tmp_file, const char *to_root, bool use_bitmap);
|
||||
|
||||
static void
|
||||
merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
|
||||
@ -439,6 +440,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
|
||||
*dest_externals = NULL;
|
||||
|
||||
parray *result_filelist = NULL;
|
||||
bool use_bitmap = true;
|
||||
// size_t total_in_place_merge_bytes = 0;
|
||||
|
||||
pthread_t *threads = NULL;
|
||||
@ -601,6 +603,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
|
||||
if (full_externals && dest_externals)
|
||||
reorder_external_dirs(full_backup, full_externals, dest_externals);
|
||||
|
||||
if (parse_program_version(dest_backup->program_version) < 20300)
|
||||
use_bitmap = false;
|
||||
|
||||
/* Setup threads */
|
||||
for (i = 0; i < parray_num(dest_backup->files); i++)
|
||||
{
|
||||
@ -639,6 +644,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
|
||||
|
||||
arg->compression_match = compression_match;
|
||||
arg->program_version_match = program_version_match;
|
||||
arg->use_bitmap = use_bitmap;
|
||||
/* By default there are some error */
|
||||
arg->ret = 1;
|
||||
|
||||
@ -1028,7 +1034,8 @@ merge_files(void *arg)
|
||||
arguments->full_backup,
|
||||
arguments->dest_backup,
|
||||
dest_file, tmp_file,
|
||||
arguments->full_database_dir);
|
||||
arguments->full_database_dir,
|
||||
arguments->use_bitmap);
|
||||
else
|
||||
merge_non_data_file(arguments->parent_chain,
|
||||
arguments->full_backup,
|
||||
@ -1128,7 +1135,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external,
|
||||
void
|
||||
merge_data_file(parray *parent_chain, pgBackup *full_backup,
|
||||
pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file,
|
||||
const char *full_database_dir)
|
||||
const char *full_database_dir, bool use_bitmap)
|
||||
{
|
||||
FILE *out = NULL;
|
||||
char *buffer = pgut_malloc(STDIO_BUFSIZE);
|
||||
@ -1154,7 +1161,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
|
||||
setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE);
|
||||
|
||||
/* restore file into temp file */
|
||||
tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1);
|
||||
tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap);
|
||||
fclose(out);
|
||||
pg_free(buffer);
|
||||
|
||||
|
@ -124,11 +124,11 @@ typedef enum CompressAlg
|
||||
|
||||
typedef enum ForkName
|
||||
{
|
||||
VM,
|
||||
FSM,
|
||||
CFM,
|
||||
INIT,
|
||||
PTRACK
|
||||
vm,
|
||||
fsm,
|
||||
cfm,
|
||||
init,
|
||||
ptrack
|
||||
} ForkName;
|
||||
|
||||
#define INIT_FILE_CRC32(use_crc32c, crc) \
|
||||
@ -935,9 +935,10 @@ extern void backup_non_data_file_internal(const char *from_fullpath,
|
||||
bool missing_ok);
|
||||
|
||||
extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file,
|
||||
FILE *out, const char *to_fullpath);
|
||||
FILE *out, const char *to_fullpath, bool use_bitmap);
|
||||
extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
|
||||
const char *from_fullpath, const char *to_fullpath, int nblocks);
|
||||
const char *from_fullpath, const char *to_fullpath, int nblocks,
|
||||
datapagemap_t *map);
|
||||
extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
|
||||
pgFile *dest_file, FILE *out, const char *to_fullpath);
|
||||
extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
|
||||
@ -1056,4 +1057,10 @@ extern void get_header_errormsg(Page page, char **errormsg);
|
||||
extern void get_checksum_errormsg(Page page, char **errormsg,
|
||||
BlockNumber absolute_blkno);
|
||||
|
||||
extern bool
|
||||
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno);
|
||||
|
||||
extern void
|
||||
datapagemap_print_debug(datapagemap_t *map);
|
||||
|
||||
#endif /* PG_PROBACKUP_H */
|
||||
|
@ -27,6 +27,7 @@ typedef struct
|
||||
bool skip_external_dirs;
|
||||
const char *to_root;
|
||||
size_t restored_bytes;
|
||||
bool use_bitmap;
|
||||
|
||||
/*
|
||||
* Return value from the thread.
|
||||
@ -501,6 +502,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
pthread_t *threads;
|
||||
restore_files_arg *threads_args;
|
||||
bool restore_isok = true;
|
||||
bool use_bitmap = true;
|
||||
|
||||
/* fancy reporting */
|
||||
char pretty_dest_bytes[20];
|
||||
@ -560,6 +562,17 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
|
||||
}
|
||||
|
||||
/* If dest backup version is older than 2.3.0, then bitmap optimization
|
||||
* is impossible to use, because bitmap restore rely on pgFile.n_blocks,
|
||||
* which is not always available in old backups.
|
||||
*/
|
||||
if (parse_program_version(dest_backup->program_version) < 20300)
|
||||
use_bitmap = false;
|
||||
|
||||
/* There is no point in bitmap restore, when restoring a single FULL backup */
|
||||
if (parray_num(parent_chain) == 1)
|
||||
use_bitmap = false;
|
||||
|
||||
/*
|
||||
* Restore dest_backup internal directories.
|
||||
*/
|
||||
@ -645,6 +658,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
arg->dbOid_exclude_list = dbOid_exclude_list;
|
||||
arg->skip_external_dirs = params->skip_external_dirs;
|
||||
arg->to_root = pgdata_path;
|
||||
arg->use_bitmap = use_bitmap;
|
||||
threads_args[i].restored_bytes = 0;
|
||||
/* By default there are some error */
|
||||
threads_args[i].ret = 1;
|
||||
@ -859,7 +873,8 @@ restore_files(void *arg)
|
||||
setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE);
|
||||
/* Destination file is data file */
|
||||
arguments->restored_bytes += restore_data_file(arguments->parent_chain,
|
||||
dest_file, out, to_fullpath);
|
||||
dest_file, out, to_fullpath,
|
||||
arguments->use_bitmap);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -871,6 +886,9 @@ restore_files(void *arg)
|
||||
arguments->dest_backup, dest_file, out, to_fullpath);
|
||||
}
|
||||
|
||||
/* free pagemap used for restore optimization */
|
||||
pg_free(dest_file->pagemap.bitmap);
|
||||
|
||||
done:
|
||||
/* close file */
|
||||
if (fio_fclose(out) != 0)
|
||||
@ -973,6 +991,9 @@ create_recovery_conf(time_t backup_id,
|
||||
elog(ERROR, "cannot open file \"%s\": %s", path,
|
||||
strerror(errno));
|
||||
|
||||
if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1)
|
||||
elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno));
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n",
|
||||
PROGRAM_VERSION);
|
||||
|
54
src/util.c
54
src/util.c
@ -495,3 +495,57 @@ str2status(const char *status)
|
||||
|
||||
return BACKUP_STATUS_INVALID;
|
||||
}
|
||||
|
||||
bool
|
||||
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno)
|
||||
{
|
||||
int offset;
|
||||
int bitno;
|
||||
|
||||
offset = blkno / 8;
|
||||
bitno = blkno % 8;
|
||||
|
||||
/* enlarge or create bitmap if needed */
|
||||
if (map->bitmapsize <= offset)
|
||||
{
|
||||
int oldsize = map->bitmapsize;
|
||||
int newsize;
|
||||
|
||||
/*
|
||||
* The minimum to hold the new bit is offset + 1. But add some
|
||||
* headroom, so that we don't need to repeatedly enlarge the bitmap in
|
||||
* the common case that blocks are modified in order, from beginning
|
||||
* of a relation to the end.
|
||||
*/
|
||||
newsize = offset + 1;
|
||||
newsize += 10;
|
||||
|
||||
map->bitmap = pg_realloc(map->bitmap, newsize);
|
||||
|
||||
/* zero out the newly allocated region */
|
||||
memset(&map->bitmap[oldsize], 0, newsize - oldsize);
|
||||
|
||||
map->bitmapsize = newsize;
|
||||
}
|
||||
|
||||
//datapagemap_print(map);
|
||||
|
||||
/* check the bit */
|
||||
return map->bitmap[offset] & (1 << bitno);
|
||||
}
|
||||
|
||||
/*
|
||||
* A debugging aid. Prints out the contents of the page map.
|
||||
*/
|
||||
void
|
||||
datapagemap_print_debug(datapagemap_t *map)
|
||||
{
|
||||
datapagemap_iterator_t *iter;
|
||||
BlockNumber blocknum;
|
||||
|
||||
iter = datapagemap_iterate(map);
|
||||
while (datapagemap_next(iter, &blocknum))
|
||||
elog(INFO, " block %u", blocknum);
|
||||
|
||||
pg_free(iter);
|
||||
}
|
||||
|
@ -629,15 +629,15 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
self.set_archiving(backup_dir, 'node', node)
|
||||
node.slow_start()
|
||||
|
||||
node.pgbench_init(scale=3)
|
||||
node.pgbench_init(scale=5)
|
||||
|
||||
# Take FULL BACKUPs
|
||||
backup_id_a = self.backup_node(backup_dir, 'node', node)
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
backup_id_b = self.backup_node(backup_dir, 'node', node)
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
# Change FULL B backup status to ERROR
|
||||
@ -648,7 +648,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
|
||||
pgdata_a1 = self.pgdata_content(node.data_dir)
|
||||
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
# PAGEa1 OK
|
||||
@ -666,20 +666,20 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
|
||||
page_id_b1 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
page_id_b2 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
page_id_b3 = self.backup_node(
|
||||
backup_dir, 'node', node, backup_type='page')
|
||||
pgdata_b3 = self.pgdata_content(node.data_dir)
|
||||
|
||||
pgbench = node.pgbench(options=['-t', '10', '-c', '2'])
|
||||
pgbench = node.pgbench(options=['-t', '20', '-c', '1'])
|
||||
pgbench.wait()
|
||||
|
||||
# PAGEb3 OK
|
||||
|
Loading…
Reference in New Issue
Block a user