From 8cbbce0b152e99885545b6895d4ed971c7fa1df6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Jun 2020 13:47:25 +0300 Subject: [PATCH] [Issue #228] drop old backup_data_file --- src/backup.c | 2 +- src/data.c | 331 +-------------------------------------------- src/merge.c | 2 +- src/pg_probackup.h | 5 - 4 files changed, 3 insertions(+), 337 deletions(-) diff --git a/src/backup.c b/src/backup.c index 57bde384..d87df4b5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2145,7 +2145,7 @@ backup_files(void *arg) /* backup file */ if (file->is_datafile && !file->is_cfs) { - backup_data_file_new(&(arguments->conn_arg), file, from_fullpath, to_fullpath, + backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, diff --git a/src/data.c b/src/data.c index c4ddcc98..553a32d7 100644 --- a/src/data.c +++ b/src/data.c @@ -540,7 +540,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, * backup with special header. */ void -backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, +backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, @@ -698,335 +698,6 @@ cleanup: pg_free(headers); } -/* - * Backup data file in the from_root directory to the to_root directory with - * same relative path. If prev_backup_start_lsn is not NULL, only pages with - * higher lsn will be copied. - * Not just copy file, but read it block by block (use bitmap in case of - * incremental backup), validate checksum, optionally compress and write to - * backup with special header. - */ -void -backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok) -{ - FILE *in = NULL; - FILE *out = NULL; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; /* number of blocks in source file */ - BlockNumber n_blocks_skipped = 0; - int rc; - char curr_page[BLCKSZ]; - bool use_pagemap; - datapagemap_iterator_t *iter = NULL; - - /* headers */ - int hdr_num = -1; - BackupPageHeader2 *headers = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; - - /* sanity */ - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; - - /* set n_blocks for a file */ - file->n_blocks = nblocks; - - /* - * Skip unchanged file only if it exists in previous backup. - * This way we can correctly handle null-sized files which are - * not tracked by pagemap and thus always marked as unchanged. - */ - if ((backup_mode == BACKUP_MODE_DIFF_PAGE || - backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev && !file->pagemap_isabsent) - { - /* - * There are no changed blocks since last backup. We want to make - * incremental backup, so we should exit. - */ - file->write_size = BYTES_INVALID; - return; - } - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); - - /* - * Read each page, verify checksum and write it to backup. - * If page map is empty or file is not present in previous backup - * backup all pages of the relation. - * - * In PTRACK 1.x there was a problem - * of data files with missing _ptrack map. - * Such files should be fully copied. - */ - - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; - - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - char *errmsg = NULL; - BlockNumber err_blknum = 0; - - int rc = fio_send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, - /* variables for error reporting */ - &err_blknum, &errmsg, &headers); - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Failed to open for reading remote file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Failed to read from remote file \"%s\"", from_fullpath); - } - - file->read_size = rc * BLCKSZ; - pg_free(errmsg); - - } - /* Local mode */ - else - { - uint cur_pos_out = 0; - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - - /* In all other cases throw an error */ - elog(ERROR, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - /* Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - - while (blknum < nblocks) - { - PageState page_st; - rc = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath, &page_st); - - if (rc == PageIsTruncated) - break; - - /* TODO: remove */ - else if (rc == SkipCurrentPage) - n_blocks_skipped++; - - else if (rc == PageIsOk) - { - - /* open local backup file for write */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - - hdr_num++; - - if (!headers) - headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); - - headers[hdr_num].block = blknum; - headers[hdr_num].lsn = page_st.lsn; - headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = cur_pos_out; /* optimize */ - - headers[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), - rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); - - file->n_headers = hdr_num +1; - - cur_pos_out += headers[hdr_num].compressed_size; - } - /* TODO: handle PageIsCorrupted, currently it is done in prepare_page */ - else - Assert(false); - - - file->read_size += BLCKSZ; - - /* next block */ - if (use_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } - } - - pg_free(file->pagemap.bitmap); - pg_free(iter); - - /* refresh n_blocks for FULL and DELTA */ - if (backup_mode == BACKUP_MODE_FULL || - backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; - - /* Determine that file didn`t changed in case of incremental backup */ - if (backup_mode != BACKUP_MODE_FULL && - file->exists_in_prev && - file->write_size == 0 && - file->n_blocks > 0) - { - file->write_size = BYTES_INVALID; - } - -cleanup: - /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); - - /* close local input file */ - if (in && fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* close local output file */ - if (out && fclose(out)) - elog(ERROR, "Cannot close the backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - pg_free(in_buf); - pg_free(out_buf); - - /* handle hdr */ - /* TODO: move in separate function */ - if (headers && file->n_headers > 0) - { - size_t hdr_size; - char to_fullpath_hdr[MAXPGPATH]; - - snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); - - out = fopen(to_fullpath_hdr, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - hdr_size = file->n_headers * sizeof(BackupPageHeader2); - -// elog(INFO, "Size: %lu, aligh: %lu", hdr_size, MAXALIGN(hdr_size)); -// elog(INFO, "checksum: %u, lsn: %lu", headers[file->n_headers-1].checksum, headers[file->n_headers-1].lsn); -// elog(INFO, "POS: %u", headers[file->n_headers-1].pos); -// elog(INFO, "blknum: %u", headers[file->n_headers-1].block); -// elog(INFO, "size: %u", headers[file->n_headers-1].compressed_size); - - if (fwrite(headers, 1, hdr_size, out) != hdr_size) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno)); - - if (fclose(out)) - elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); - - /* TODO: fsync */ - -// elog(INFO, "n_headers: %u", file->n_headers); - } - - pg_free(headers); -} - /* * Backup non data file * We do not apply compression to this file. diff --git a/src/merge.c b/src/merge.c index c0709e3e..fbc271b7 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1186,7 +1186,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, were n_blocks is missing. */ - backup_data_file_new(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3f3e90a3..5e250425 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -964,11 +964,6 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok); -extern void backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time,