db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge branch 'issue_169' into issue_174

Browse Source
This commit is contained in:
Grigory Smolkin
2020-03-03 19:44:57 +03:00
parent 61b25eb1e5 e9c31759c5
commit 0723eefc78
14 changed files with 229 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backup.c
+13 -10
View File
@@ -575,6 +575,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
{
parray *xlog_files_list;
char pg_xlog_path[MAXPGPATH];
char wal_full_path[MAXPGPATH];
/* Scan backup PG_XLOG_DIR */
xlog_files_list = parray_new();
@@ -586,11 +587,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
for (i = 0; i < parray_num(xlog_files_list); i++)
{
pgFile *file = (pgFile *) parray_get(xlog_files_list, i);
join_path_components(wal_full_path, pg_xlog_path, file->rel_path);
if (S_ISREG(file->mode))
{
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->crc = pgFileGetCRC(wal_full_path, true, false);
file->write_size = file->size;
}
/* Remove file path root prefix*/
if (strstr(file->path, database_path) == file->path)
@@ -1805,10 +1808,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
{
file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0,
FIO_BACKUP_HOST);
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->uncompressed_size = file->read_size;
file->crc = pgFileGetCRC(backup_label, true, false);
file->write_size = file->size;
file->uncompressed_size = file->size;
free(file->path);
file->path = strdup(PG_BACKUP_LABEL_FILE);
parray_append(backup_files_list, file);
@@ -1854,9 +1858,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
FIO_BACKUP_HOST);
if (S_ISREG(file->mode))
{
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->crc = pgFileGetCRC(tablespace_map, true, false);
file->write_size = file->size;
}
free(file->path);
file->path = strdup(PG_TABLESPACE_MAP_FILE);
src/data.c
+8 -4
View File
@@ -613,7 +613,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
}
if (!fio_is_remote_file(in))
setbuf(in, in_buffer);
setbuffer(in, in_buffer, STDIO_BUFSIZE);
/* open backup file for write */
out = fopen(to_fullpath, PG_BINARY_W);
@@ -621,7 +621,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
elog(ERROR, "Cannot open backup file \"%s\": %s",
to_fullpath, strerror(errno));
setbuf(out, out_buffer);
setbuffer(out, out_buffer, STDIO_BUFSIZE);
/* update file permission */
if (chmod(to_fullpath, FILE_PERMISSION) == -1)
@@ -844,6 +844,10 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
if (tmp_file->write_size == BYTES_INVALID)
continue;
/* If file was truncated in intermediate backup,
* it is ok not to truncate it now, because old blocks will be
* overwritten by new blocks from next backup.
*/
if (tmp_file->write_size == 0)
continue;
@@ -859,7 +863,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
setbuf(in, buffer);
setbuffer(in, buffer, STDIO_BUFSIZE);
/*
* Restore the file.
@@ -1149,7 +1153,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath,
strerror(errno));
setbuf(in, buffer);
setbuffer(in, buffer, STDIO_BUFSIZE);
/* do actual work */
restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath);
src/dir.c
+2 -4
View File
@@ -259,10 +259,9 @@ delete_file:
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
* TODO: add decompression option.
*/
pg_crc32
pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
pg_crc32 crc = 0;
@@ -321,7 +320,6 @@ pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
* TODO: add decompression option.
*/
pg_crc32
pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
@@ -1842,7 +1840,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
FIO_BACKUP_HOST);
pfree(file->path);
file->path = pgut_strdup(DATABASE_MAP);
file->crc = pgFileGetCRCnew(database_map_path, true, false);
file->crc = pgFileGetCRC(database_map_path, true, false);
file->write_size = file->read_size;
file->uncompressed_size = file->read_size;
src/fetch.c
+4 -73
View File
@@ -13,67 +13,6 @@
#include <sys/stat.h>
#include <unistd.h>
/*
* Read a file into memory.
* The file contents are returned in a malloc'd buffer, and *filesize
* is set to the length of the file.
*
* The returned buffer is always zero-terminated; the size of the returned
* buffer is actually *filesize + 1. That's handy when reading a text file.
* This function can be used to read binary files as well, you can just
* ignore the zero-terminator in that case.
*
*/
char *
slurpFileFullPath(const char *from_fullpath, size_t *filesize, bool safe, fio_location location)
{
int fd;
char *buffer;
int len;
struct stat statbuf;
if ((fd = fio_open(from_fullpath, O_RDONLY | PG_BINARY, location)) == -1)
{
if (safe)
return NULL;
else
elog(ERROR, "Could not open file \"%s\" for reading: %s",
from_fullpath, strerror(errno));
}
if (fio_fstat(fd, &statbuf) < 0)
{
if (safe)
return NULL;
else
elog(ERROR, "Could not stat file \"%s\": %s",
from_fullpath, strerror(errno));
}
len = statbuf.st_size;
buffer = pg_malloc(len + 1);
if (fio_read(fd, buffer, len) != len)
{
if (safe)
return NULL;
else
elog(ERROR, "Could not read file \"%s\": %s\n",
from_fullpath, strerror(errno));
}
fio_close(fd);
/* Zero-terminate the buffer. */
buffer[len] = '\0';
if (filesize)
*filesize = len;
return buffer;
}
/*
* Read a file into memory. The file to be read is <datadir>/<path>.
* The file contents are returned in a malloc'd buffer, and *filesize
@@ -93,23 +32,15 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi
struct stat statbuf;
char fullpath[MAXPGPATH];
int len;
snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
if (fio_access(fullpath, R_OK, location) != 0)
{
if (safe)
return NULL;
else
elog(ERROR, "could not open file \"%s\" for reading: %s",
fullpath, strerror(errno));
}
join_path_components(fullpath, datadir, path);
if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1)
{
if (safe)
return NULL;
else
elog(ERROR, "could not open file \"%s\" for reading: %s",
elog(ERROR, "Could not open file \"%s\" for reading: %s",
fullpath, strerror(errno));
}
@@ -118,7 +49,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi
if (safe)
return NULL;
else
elog(ERROR, "could not open file \"%s\" for reading: %s",
elog(ERROR, "Could not stat file \"%s\": %s",
fullpath, strerror(errno));
}
@@ -130,7 +61,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi
if (safe)
return NULL;
else
elog(ERROR, "could not read file \"%s\": %s\n",
elog(ERROR, "Could not read file \"%s\": %s\n",
fullpath, strerror(errno));
}
src/help.c
+6
View File
@@ -119,6 +119,7 @@ help_pg_probackup(void)
printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n"));
printf(_(" [--no-validate] [--skip-block-validation]\n"));
printf(_(" [--external-dirs=external-directories-paths]\n"));
printf(_(" [--no-sync]\n"));
printf(_(" [--log-level-console=log-level-console]\n"));
printf(_(" [--log-level-file=log-level-file]\n"));
printf(_(" [--log-filename=log-filename]\n"));
@@ -156,6 +157,7 @@ help_pg_probackup(void)
printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n"));
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n"));
printf(_(" [--no-sync]\n"));
printf(_(" [--db-include | --db-exclude]\n"));
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
@@ -257,6 +259,7 @@ help_backup(void)
printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n"));
printf(_(" [--no-validate] [--skip-block-validation]\n"));
printf(_(" [-E external-directories-paths]\n"));
printf(_(" [--no-sync]\n"));
printf(_(" [--log-level-console=log-level-console]\n"));
printf(_(" [--log-level-file=log-level-file]\n"));
printf(_(" [--log-filename=log-filename]\n"));
@@ -295,6 +298,7 @@ help_backup(void)
printf(_(" -E --external-dirs=external-directories-paths\n"));
printf(_(" backup some directories not from pgdata \n"));
printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n"));
printf(_(" --no-sync do not sync backed up files to disk\n"));
printf(_("\n Logging options:\n"));
printf(_(" --log-level-console=log-level-console\n"));
@@ -391,6 +395,7 @@ help_restore(void)
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--skip-external-dirs]\n"));
printf(_(" [--restore-command=cmdline]\n"));
printf(_(" [--no-sync]\n"));
printf(_(" [--db-include dbname | --db-exclude dbname]\n"));
printf(_(" [--remote-proto] [--remote-host]\n"));
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
@@ -433,6 +438,7 @@ help_restore(void)
printf(_(" relocate the external directory from OLDDIR to NEWDIR\n"));
printf(_(" --skip-external-dirs do not restore all external directories\n"));
printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n"));
printf(_(" --no-sync do not sync restored files to disk\n"));
printf(_("\n Partial restore options:\n"));
printf(_(" --db-include dbname restore only specified databases\n"));
src/merge.c
+10 -10
View File
@@ -427,13 +427,13 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
/* for fancy reporting */
time_t end_time;
char pretty_time[20];
/* in-place flags */
/* in-place merge flags */
bool compression_match = false;
bool program_version_match = false;
/* It's redundant to check block checksumms during merge */
skip_block_validation = true;
/* Handle corner cases missing destination backup */
/* Handle corner cases of missing destination backup */
if (dest_backup == NULL &&
full_backup->status == BACKUP_STATUS_MERGED)
goto merge_rename;
@@ -740,12 +740,11 @@ merge_delete:
}
/*
* If we crash now, automatic rerun of failed merge will be impossible.
* The user must have to manually change start_time of FULL backup
* to start_time of destination backup:
* PAGE2 DELETED
* PAGE1 DELETED
* FULL MERGED
* If we crash now, automatic rerun of failed merge is still possible:
* The user should start merge with full backup ID as an argument to option '-i'.
*/
merge_rename:
@@ -891,7 +890,7 @@ merge_files(void *arg)
*
* Case 2:
* in this case in place merge is possible:
* 0 PAGE; file, size BYTES_INVALID (should not be possible)
* 0 PAGE; file, size 0
* 1 PAGE; file, size 0
* 2 FULL; file, size 100500
*
@@ -901,11 +900,11 @@ merge_files(void *arg)
* 1 PAGE; file, size 100501
* 2 FULL; file, size 100500
*
* Case 4:
* Case 4 (good candidate for future optimization):
* in this case in place merge is impossible:
* 0 PAGE; file, size BYTES_INVALID
* 1 PAGE; file, size 100501
* 2 FULL; file, missing
* 2 FULL; file, not exists yet
*/
in_place = true;
@@ -1107,7 +1106,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
if (out == NULL)
elog(ERROR, "Cannot open merge target file \"%s\": %s",
to_fullpath_tmp1, strerror(errno));
setbuf(out, buffer);
setbuffer(out, buffer, STDIO_BUFSIZE);
/* restore file into temp file */
tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1);
@@ -1138,6 +1137,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
if (tmp_file->write_size == 0)
return;
/* sync second temp file to disk */
if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Cannot sync merge temp file \"%s\": %s",
to_fullpath_tmp2, strerror(errno));
@@ -1242,7 +1242,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
backup_non_data_file(tmp_file, NULL, from_fullpath,
to_fullpath_tmp, BACKUP_MODE_FULL, 0, false);
/* TODO: --no-sync support */
/* sync temp file to disk */
if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Cannot sync merge temp file \"%s\": %s",
to_fullpath_tmp, strerror(errno));
src/pg_probackup.c
+14 -11
View File
@@ -153,28 +153,29 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg);
/*
* Short name should be non-printable ASCII character.
* Use values between 128 and 255.
*/
static ConfigOption cmd_options[] =
{
/* directory options */
{ 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT },
{ 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT },
{ 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT },
/* common options */
{ 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT },
{ 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT },
{ 'b', 122, "progress", &progress, SOURCE_CMD_STRICT },
{ 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT },
{ 'b', 132, "progress", &progress, SOURCE_CMD_STRICT },
{ 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT },
{ 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT },
{ 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT },
/* backup options */
{ 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT },
{ 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT },
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT },
{ 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT },
{ 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT },
{ 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT },
{ 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT },
{ 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT },
{ 'b', 235, "merge-expired", &merge_expired, SOURCE_CMD_STRICT },
{ 'b', 237, "dry-run", &dry_run, SOURCE_CMD_STRICT },
{ 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT },
{ 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT },
{ 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT },
{ 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT },
{ 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT },
/* restore options */
{ 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT },
{ 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT },
@@ -220,7 +221,9 @@ static ConfigOption cmd_options[] =
{ 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value},
{ 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT },
/* options for backward compatibility */
/* options for backward compatibility
* TODO: remove in 3.0.0
*/
{ 's', 136, "time", &target_time, SOURCE_CMD_STRICT },
{ 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT },
{ 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT },
src/pg_probackup.h
+3 -7
View File
@@ -718,9 +718,6 @@ extern char *slurpFile(const char *datadir,
size_t *filesize,
bool safe,
fio_location location);
extern char *slurpFileFullPath(const char *from_fullpath,
size_t *filesize, bool safe,
fio_location location);
extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize);
/* in help.c */
@@ -835,10 +832,9 @@ extern pgFile *pgFileInit(const char *path, const char *rel_path);
extern void pgFileDelete(pgFile *file, const char *full_path);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
bool raise_on_deleted, size_t *bytes_read, fio_location location);
extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c);
extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c);
extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool missing_ok, bool use_crc32c);
extern int pgFileCompareName(const void *f1, const void *f2);
extern int pgFileComparePath(const void *f1, const void *f2);
src/restore.c
+8 -5
View File
@@ -441,7 +441,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
base36enc(dest_backup->start_time), status2str(dest_backup->status));
/* We ensured that all backups are valid, now restore if required
* TODO: before restore - lock entire parent chain
*/
if (params->is_restore)
{
@@ -555,7 +554,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
else
backup->files = dest_files;
/* this sorting is important */
/*
* this sorting is important, because we rely on it to find
* destination file in intermediate backups file lists
* using bsearch.
*/
parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
}
@@ -622,8 +625,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) *
num_threads);
/* Restore files into target directory */
if (dest_backup->stream)
dest_bytes = dest_backup->pgdata_bytes + dest_backup->wal_bytes;
else
@@ -633,6 +634,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes);
time(&start_time);
thread_interrupted = false;
/* Restore files into target directory */
for (i = 0; i < num_threads; i++)
{
restore_files_arg *arg = &(threads_args[i]);
@@ -851,7 +854,7 @@ restore_files(void *arg)
goto done;
if (!fio_is_remote_file(out))
setbuf(out, buffer);
setbuffer(out, buffer, STDIO_BUFSIZE);
/* Restore destination file */
if (dest_file->is_datafile && !dest_file->is_cfs)
src/util.c
+1 -1
View File
@@ -390,7 +390,7 @@ copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
char *buffer;
size_t size;
buffer = slurpFileFullPath(from_fullpath, &size, false, from_location);
buffer = slurpFile(from_fullpath, "", &size, false, from_location);
digestControlFile(&ControlFile, buffer, size);
src/utils/file.c
+3 -2
View File
@@ -553,6 +553,7 @@ int fio_pread(FILE* f, void* buf, off_t offs)
}
else
{
/* For local file, opened by fopen, we should use stdio operations */
int rc;
rc = fseek(f, offs, SEEK_SET);
@@ -939,7 +940,7 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decomp
if (decompress)
return pgFileGetCRCgz(file_path, true, true);
else
return pgFileGetCRCnew(file_path, true, true);
return pgFileGetCRC(file_path, true, true);
}
}
@@ -1653,7 +1654,7 @@ void fio_communicate(int in, int out)
if (hdr.arg == 1)
crc = pgFileGetCRCgz(buf, true, true);
else
crc = pgFileGetCRCnew(buf, true, true);
crc = pgFileGetCRC(buf, true, true);
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
break;
default:
src/validate.c
+5 -3
View File
@@ -264,6 +264,7 @@ pgBackupValidateFiles(void *arg)
*/
if (file->write_size == BYTES_INVALID)
{
/* TODO: lookup corresponding merge bug */
if (arguments->backup_mode == BACKUP_MODE_FULL)
{
/* It is illegal for file in FULL backup to have BYTES_INVALID */
@@ -276,10 +277,11 @@ pgBackupValidateFiles(void *arg)
continue;
}
/* no point in trying to open empty or non-changed files */
if (file->write_size <= 0)
/* no point in trying to open empty file */
if (file->write_size == 0)
continue;
/* TODO: it is redundant to check file existence using stat */
if (stat(file->path, &st) == -1)
{
if (errno == ENOENT)
@@ -326,7 +328,7 @@ pgBackupValidateFiles(void *arg)
crc = pgFileGetCRC(file->path,
arguments->backup_version <= 20021 ||
arguments->backup_version >= 20025,
true, NULL, FIO_LOCAL_HOST);
false);
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
tests/compatibility.py
+77
View File
@@ -688,3 +688,80 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_page_vacuum_truncate(self):
"""
make node, create table, take full backup,
delete all data, vacuum relation,
take page backup, insert some data,
take second page backup,
restore latest page backup using new binary
and check data correctness
old binary should be 2.2.x version
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir, old_binary=True)
self.add_instance(backup_dir, 'node', node, old_binary=True)
self.set_archiving(backup_dir, 'node', node, old_binary=True)
node.slow_start()
node.safe_psql(
"postgres",
"create sequence t_seq; "
"create table t_heap as select i as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1024) i")
node.safe_psql(
"postgres",
"vacuum t_heap")
self.backup_node(backup_dir, 'node', node, old_binary=True)
node.safe_psql(
"postgres",
"delete from t_heap")
node.safe_psql(
"postgres",
"vacuum t_heap")
self.backup_node(
backup_dir, 'node', node, backup_type='page', old_binary=True)
node.safe_psql(
"postgres",
"insert into t_heap select i as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1) i")
self.backup_node(
backup_dir, 'node', node, backup_type='page', old_binary=True)
pgdata = self.pgdata_content(node.data_dir)
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
self.restore_node(backup_dir, 'node', node_restored)
# Physical comparison
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
self.set_auto_conf(node_restored, {'port': node_restored.port})
node_restored.slow_start()
# Clean after yourself
self.del_test_dir(module_name, fname)
tests/page.py
+75
View File
@@ -54,6 +54,7 @@ class PageTest(ProbackupTest, unittest.TestCase):
self.backup_node(backup_dir, 'node', node)
# TODO: make it dynamic
node.safe_psql(
"postgres",
"delete from t_heap where ctid >= '(11,0)'")
@@ -101,6 +102,80 @@ class PageTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_page_vacuum_truncate_1(self):
"""
make node, create table, take full backup,
delete all data, vacuum relation,
take page backup, insert some data,
take second page backup and check data correctness
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir)
self.add_instance(backup_dir, 'node', node)
self.set_archiving(backup_dir, 'node', node)
node.slow_start()
node.safe_psql(
"postgres",
"create sequence t_seq; "
"create table t_heap as select i as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1024) i")
node.safe_psql(
"postgres",
"vacuum t_heap")
self.backup_node(backup_dir, 'node', node)
node.safe_psql(
"postgres",
"delete from t_heap")
node.safe_psql(
"postgres",
"vacuum t_heap")
self.backup_node(
backup_dir, 'node', node, backup_type='page')
node.safe_psql(
"postgres",
"insert into t_heap select i as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1) i")
self.backup_node(
backup_dir, 'node', node, backup_type='page')
pgdata = self.pgdata_content(node.data_dir)
node_restored = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node_restored'))
node_restored.cleanup()
self.restore_node(backup_dir, 'node', node_restored)
# Physical comparison
pgdata_restored = self.pgdata_content(node_restored.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
self.set_auto_conf(node_restored, {'port': node_restored.port})
node_restored.slow_start()
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_page_stream(self):
"""