mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2026-06-21 01:34:15 +02:00
[Issue #228] drop old backup_data_file
This commit is contained in:
Grigory Smolkin
+1
-1
@@ -2145,7 +2145,7 @@ backup_files(void *arg)
|
||||
/* backup file */
|
||||
if (file->is_datafile && !file->is_cfs)
|
||||
{
|
||||
backup_data_file_new(&(arguments->conn_arg), file, from_fullpath, to_fullpath,
|
||||
backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath,
|
||||
arguments->prev_start_lsn,
|
||||
current.backup_mode,
|
||||
instance_config.compress_alg,
|
||||
|
||||
+1
-330
@@ -540,7 +540,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
|
||||
* backup with special header.
|
||||
*/
|
||||
void
|
||||
backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file,
|
||||
backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
|
||||
const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
|
||||
CompressAlg calg, int clevel, uint32 checksum_version,
|
||||
@@ -698,335 +698,6 @@ cleanup:
|
||||
pg_free(headers);
|
||||
}
|
||||
|
||||
/*
|
||||
* Backup data file in the from_root directory to the to_root directory with
|
||||
* same relative path. If prev_backup_start_lsn is not NULL, only pages with
|
||||
* higher lsn will be copied.
|
||||
* Not just copy file, but read it block by block (use bitmap in case of
|
||||
* incremental backup), validate checksum, optionally compress and write to
|
||||
* backup with special header.
|
||||
*/
|
||||
void
|
||||
backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
|
||||
const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
|
||||
CompressAlg calg, int clevel, uint32 checksum_version,
|
||||
int ptrack_version_num, const char *ptrack_schema, bool missing_ok)
|
||||
{
|
||||
FILE *in = NULL;
|
||||
FILE *out = NULL;
|
||||
BlockNumber blknum = 0;
|
||||
BlockNumber nblocks = 0; /* number of blocks in source file */
|
||||
BlockNumber n_blocks_skipped = 0;
|
||||
int rc;
|
||||
char curr_page[BLCKSZ];
|
||||
bool use_pagemap;
|
||||
datapagemap_iterator_t *iter = NULL;
|
||||
|
||||
/* headers */
|
||||
int hdr_num = -1;
|
||||
BackupPageHeader2 *headers = NULL;
|
||||
|
||||
/* stdio buffers */
|
||||
char *in_buf = NULL;
|
||||
char *out_buf = NULL;
|
||||
|
||||
/* sanity */
|
||||
if (file->size % BLCKSZ != 0)
|
||||
elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size);
|
||||
|
||||
/*
|
||||
* Compute expected number of blocks in the file.
|
||||
* NOTE This is a normal situation, if the file size has changed
|
||||
* since the moment we computed it.
|
||||
*/
|
||||
nblocks = file->size/BLCKSZ;
|
||||
|
||||
/* set n_blocks for a file */
|
||||
file->n_blocks = nblocks;
|
||||
|
||||
/*
|
||||
* Skip unchanged file only if it exists in previous backup.
|
||||
* This way we can correctly handle null-sized files which are
|
||||
* not tracked by pagemap and thus always marked as unchanged.
|
||||
*/
|
||||
if ((backup_mode == BACKUP_MODE_DIFF_PAGE ||
|
||||
backup_mode == BACKUP_MODE_DIFF_PTRACK) &&
|
||||
file->pagemap.bitmapsize == PageBitmapIsEmpty &&
|
||||
file->exists_in_prev && !file->pagemap_isabsent)
|
||||
{
|
||||
/*
|
||||
* There are no changed blocks since last backup. We want to make
|
||||
* incremental backup, so we should exit.
|
||||
*/
|
||||
file->write_size = BYTES_INVALID;
|
||||
return;
|
||||
}
|
||||
|
||||
/* reset size summary */
|
||||
file->read_size = 0;
|
||||
file->write_size = 0;
|
||||
file->uncompressed_size = 0;
|
||||
INIT_FILE_CRC32(true, file->crc);
|
||||
|
||||
/*
|
||||
* Read each page, verify checksum and write it to backup.
|
||||
* If page map is empty or file is not present in previous backup
|
||||
* backup all pages of the relation.
|
||||
*
|
||||
* In PTRACK 1.x there was a problem
|
||||
* of data files with missing _ptrack map.
|
||||
* Such files should be fully copied.
|
||||
*/
|
||||
|
||||
if (file->pagemap.bitmapsize == PageBitmapIsEmpty ||
|
||||
file->pagemap_isabsent || !file->exists_in_prev ||
|
||||
!file->pagemap.bitmap)
|
||||
use_pagemap = false;
|
||||
else
|
||||
use_pagemap = true;
|
||||
|
||||
/* Remote mode */
|
||||
if (fio_is_remote(FIO_DB_HOST))
|
||||
{
|
||||
char *errmsg = NULL;
|
||||
BlockNumber err_blknum = 0;
|
||||
|
||||
int rc = fio_send_pages(to_fullpath, from_fullpath, file,
|
||||
/* send prev backup START_LSN */
|
||||
backup_mode == BACKUP_MODE_DIFF_DELTA &&
|
||||
file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr,
|
||||
calg, clevel, checksum_version,
|
||||
/* send pagemap if any */
|
||||
use_pagemap ? &file->pagemap : NULL,
|
||||
/* variables for error reporting */
|
||||
&err_blknum, &errmsg, &headers);
|
||||
|
||||
/* check for errors */
|
||||
if (rc == FILE_MISSING)
|
||||
{
|
||||
elog(LOG, "File \"%s\" is not found", from_fullpath);
|
||||
file->write_size = FILE_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
else if (rc == WRITE_FAILED)
|
||||
elog(ERROR, "Cannot write block %u of \"%s\": %s",
|
||||
err_blknum, to_fullpath, strerror(errno));
|
||||
|
||||
else if (rc == PAGE_CORRUPTION)
|
||||
{
|
||||
if (errmsg)
|
||||
elog(ERROR, "Corruption detected in file \"%s\", block %u: %s",
|
||||
from_fullpath, err_blknum, errmsg);
|
||||
else
|
||||
elog(ERROR, "Corruption detected in file \"%s\", block %u",
|
||||
from_fullpath, err_blknum);
|
||||
}
|
||||
/* OPEN_FAILED and READ_FAILED */
|
||||
else if (rc == OPEN_FAILED)
|
||||
{
|
||||
if (errmsg)
|
||||
elog(ERROR, "%s", errmsg);
|
||||
else
|
||||
elog(ERROR, "Failed to open for reading remote file \"%s\"", from_fullpath);
|
||||
}
|
||||
else if (rc == READ_FAILED)
|
||||
{
|
||||
if (errmsg)
|
||||
elog(ERROR, "%s", errmsg);
|
||||
else
|
||||
elog(ERROR, "Failed to read from remote file \"%s\"", from_fullpath);
|
||||
}
|
||||
|
||||
file->read_size = rc * BLCKSZ;
|
||||
pg_free(errmsg);
|
||||
|
||||
}
|
||||
/* Local mode */
|
||||
else
|
||||
{
|
||||
uint cur_pos_out = 0;
|
||||
/* open source file for read */
|
||||
in = fopen(from_fullpath, PG_BINARY_R);
|
||||
if (in == NULL)
|
||||
{
|
||||
/*
|
||||
* If file is not found, this is not en error.
|
||||
* It could have been deleted by concurrent postgres transaction.
|
||||
*/
|
||||
if (errno == ENOENT)
|
||||
{
|
||||
if (missing_ok)
|
||||
{
|
||||
elog(LOG, "File \"%s\" is not found", from_fullpath);
|
||||
file->write_size = FILE_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
else
|
||||
elog(ERROR, "File \"%s\" is not found", from_fullpath);
|
||||
}
|
||||
|
||||
/* In all other cases throw an error */
|
||||
elog(ERROR, "Cannot open file \"%s\": %s",
|
||||
from_fullpath, strerror(errno));
|
||||
}
|
||||
|
||||
/* Enable stdio buffering for local input file,
|
||||
* unless the pagemap is involved, which
|
||||
* imply a lot of random access.
|
||||
*/
|
||||
|
||||
if (use_pagemap)
|
||||
{
|
||||
iter = datapagemap_iterate(&file->pagemap);
|
||||
datapagemap_next(iter, &blknum); /* set first block */
|
||||
|
||||
setvbuf(in, NULL, _IONBF, BUFSIZ);
|
||||
}
|
||||
else
|
||||
{
|
||||
in_buf = pgut_malloc(STDIO_BUFSIZE);
|
||||
setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE);
|
||||
}
|
||||
|
||||
while (blknum < nblocks)
|
||||
{
|
||||
PageState page_st;
|
||||
rc = prepare_page(conn_arg, file, prev_backup_start_lsn,
|
||||
blknum, in, backup_mode, curr_page,
|
||||
true, checksum_version,
|
||||
ptrack_version_num, ptrack_schema,
|
||||
from_fullpath, &page_st);
|
||||
|
||||
if (rc == PageIsTruncated)
|
||||
break;
|
||||
|
||||
/* TODO: remove */
|
||||
else if (rc == SkipCurrentPage)
|
||||
n_blocks_skipped++;
|
||||
|
||||
else if (rc == PageIsOk)
|
||||
{
|
||||
|
||||
/* open local backup file for write */
|
||||
if (!out)
|
||||
out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE);
|
||||
|
||||
hdr_num++;
|
||||
|
||||
if (!headers)
|
||||
headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2));
|
||||
else
|
||||
headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2));
|
||||
|
||||
headers[hdr_num].block = blknum;
|
||||
headers[hdr_num].lsn = page_st.lsn;
|
||||
headers[hdr_num].checksum = page_st.checksum;
|
||||
headers[hdr_num].pos = cur_pos_out; /* optimize */
|
||||
|
||||
headers[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc),
|
||||
rc, curr_page, calg, clevel,
|
||||
from_fullpath, to_fullpath);
|
||||
|
||||
file->n_headers = hdr_num +1;
|
||||
|
||||
cur_pos_out += headers[hdr_num].compressed_size;
|
||||
}
|
||||
/* TODO: handle PageIsCorrupted, currently it is done in prepare_page */
|
||||
else
|
||||
Assert(false);
|
||||
|
||||
|
||||
file->read_size += BLCKSZ;
|
||||
|
||||
/* next block */
|
||||
if (use_pagemap)
|
||||
{
|
||||
/* exit if pagemap is exhausted */
|
||||
if (!datapagemap_next(iter, &blknum))
|
||||
break;
|
||||
}
|
||||
else
|
||||
blknum++;
|
||||
}
|
||||
}
|
||||
|
||||
pg_free(file->pagemap.bitmap);
|
||||
pg_free(iter);
|
||||
|
||||
/* refresh n_blocks for FULL and DELTA */
|
||||
if (backup_mode == BACKUP_MODE_FULL ||
|
||||
backup_mode == BACKUP_MODE_DIFF_DELTA)
|
||||
file->n_blocks = file->read_size / BLCKSZ;
|
||||
|
||||
/* Determine that file didn`t changed in case of incremental backup */
|
||||
if (backup_mode != BACKUP_MODE_FULL &&
|
||||
file->exists_in_prev &&
|
||||
file->write_size == 0 &&
|
||||
file->n_blocks > 0)
|
||||
{
|
||||
file->write_size = BYTES_INVALID;
|
||||
}
|
||||
|
||||
cleanup:
|
||||
/* finish CRC calculation */
|
||||
FIN_FILE_CRC32(true, file->crc);
|
||||
|
||||
/* close local input file */
|
||||
if (in && fclose(in))
|
||||
elog(ERROR, "Cannot close the source file \"%s\": %s",
|
||||
to_fullpath, strerror(errno));
|
||||
|
||||
/* close local output file */
|
||||
if (out && fclose(out))
|
||||
elog(ERROR, "Cannot close the backup file \"%s\": %s",
|
||||
to_fullpath, strerror(errno));
|
||||
|
||||
pg_free(in_buf);
|
||||
pg_free(out_buf);
|
||||
|
||||
/* handle hdr */
|
||||
/* TODO: move in separate function */
|
||||
if (headers && file->n_headers > 0)
|
||||
{
|
||||
size_t hdr_size;
|
||||
char to_fullpath_hdr[MAXPGPATH];
|
||||
|
||||
snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath);
|
||||
|
||||
out = fopen(to_fullpath_hdr, PG_BINARY_W);
|
||||
if (out == NULL)
|
||||
elog(ERROR, "Cannot open header file \"%s\": %s",
|
||||
to_fullpath, strerror(errno));
|
||||
|
||||
/* update file permission */
|
||||
if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1)
|
||||
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
|
||||
strerror(errno));
|
||||
|
||||
hdr_size = file->n_headers * sizeof(BackupPageHeader2);
|
||||
|
||||
// elog(INFO, "Size: %lu, aligh: %lu", hdr_size, MAXALIGN(hdr_size));
|
||||
// elog(INFO, "checksum: %u, lsn: %lu", headers[file->n_headers-1].checksum, headers[file->n_headers-1].lsn);
|
||||
// elog(INFO, "POS: %u", headers[file->n_headers-1].pos);
|
||||
// elog(INFO, "blknum: %u", headers[file->n_headers-1].block);
|
||||
// elog(INFO, "size: %u", headers[file->n_headers-1].compressed_size);
|
||||
|
||||
if (fwrite(headers, 1, hdr_size, out) != hdr_size)
|
||||
elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno));
|
||||
|
||||
if (fclose(out))
|
||||
elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno));
|
||||
|
||||
/* TODO: fsync */
|
||||
|
||||
// elog(INFO, "n_headers: %u", file->n_headers);
|
||||
}
|
||||
|
||||
pg_free(headers);
|
||||
}
|
||||
|
||||
/*
|
||||
* Backup non data file
|
||||
* We do not apply compression to this file.
|
||||
|
||||
+1
-1
@@ -1186,7 +1186,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
|
||||
* 2 backups of old versions, were n_blocks is missing.
|
||||
*/
|
||||
|
||||
backup_data_file_new(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2,
|
||||
backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2,
|
||||
InvalidXLogRecPtr, BACKUP_MODE_FULL,
|
||||
dest_backup->compress_alg, dest_backup->compress_level,
|
||||
dest_backup->checksum_version, 0, NULL, false);
|
||||
|
||||
@@ -964,11 +964,6 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
|
||||
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
|
||||
CompressAlg calg, int clevel, uint32 checksum_version,
|
||||
int ptrack_version_num, const char *ptrack_schema, bool missing_ok);
|
||||
extern void backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file,
|
||||
const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
|
||||
CompressAlg calg, int clevel, uint32 checksum_version,
|
||||
int ptrack_version_num, const char *ptrack_schema, bool missing_ok);
|
||||
extern void backup_non_data_file(pgFile *file, pgFile *prev_file,
|
||||
const char *from_fullpath, const char *to_fullpath,
|
||||
BackupMode backup_mode, time_t parent_backup_time,
|
||||
|
||||
Reference in New Issue
Block a user