From 007464ae8f561409f157c96e349ce51715938a00 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@182aca00-e38e-11de-a668-6fd11605f5ce> Date: Tue, 8 Dec 2009 00:10:30 +0000 Subject: [PATCH 0001/1134] Initial directory structure. git-svn-id: http://pg-rman.googlecode.com/svn/trunk@1 182aca00-e38e-11de-a668-6fd11605f5ce From 7fbb857d03d70bf799ad66980a406a3a3eaab2cd Mon Sep 17 00:00:00 2001 From: "itagaki.takahiro" Date: Tue, 8 Dec 2009 00:21:28 +0000 Subject: [PATCH 0002/1134] First import. git-svn-id: http://pg-rman.googlecode.com/svn/trunk@2 182aca00-e38e-11de-a668-6fd11605f5ce --- COPYRIGHT | 26 + Makefile | 40 + backup.c | 1110 +++++++++++ catalog.c | 593 ++++++ data.c | 886 +++++++++ data/sample_backup/20090531/170553/backup.ini | 18 + .../20090531/170553/database/PG_VERSION | 1 + .../20090531/170553/file_arclog.txt | 0 .../20090531/170553/file_database.txt | 1 + data/sample_backup/20090601/170553/backup.ini | 18 + .../20090601/170553/database/PG_VERSION | 1 + .../20090601/170553/file_arclog.txt | 0 .../20090601/170553/file_database.txt | 1 + data/sample_backup/20090602/170553/backup.ini | 18 + data/sample_backup/20090603/170553/backup.ini | 18 + .../20090603/170553/file_arclog.txt | 0 .../20090603/170553/file_database.txt | 0 .../20090603/170553/file_srvlog.txt | 0 data/sample_backup/pg_rman.ini | 0 delete.c | 235 +++ dir.c | 567 ++++++ expected/backup_restore.out | 40 + expected/init.out | 14 + expected/option.out | 80 + expected/show_validate.out | 50 + init.c | 156 ++ parray.c | 184 ++ parray.h | 34 + pg_rman.c | 301 +++ pg_rman.h | 283 +++ pgsql_src/COPYRIGHT.pgsql_src | 24 + pgsql_src/pg_crc.c | 515 +++++ pgsql_src/pg_ctl.c | 105 + pgut/pgut-port.c | 240 +++ pgut/pgut-port.h | 71 + pgut/pgut.c | 1686 +++++++++++++++++ pgut/pgut.h | 245 +++ restore.c | 978 ++++++++++ show.c | 255 +++ sql/backup.sql | 3 + sql/backup_restore.sh | 274 +++ sql/backup_restore.sql | 2 + sql/init.sql | 4 + sql/option.sh | 80 + sql/option.sql | 1 + sql/show_validate.sql | 8 + util.c | 85 + validate.c | 183 ++ xlog.c | 91 + 49 files changed, 9525 insertions(+) create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 backup.c create mode 100644 catalog.c create mode 100644 data.c create mode 100644 data/sample_backup/20090531/170553/backup.ini create mode 100644 data/sample_backup/20090531/170553/database/PG_VERSION create mode 100644 data/sample_backup/20090531/170553/file_arclog.txt create mode 100644 data/sample_backup/20090531/170553/file_database.txt create mode 100644 data/sample_backup/20090601/170553/backup.ini create mode 100644 data/sample_backup/20090601/170553/database/PG_VERSION create mode 100644 data/sample_backup/20090601/170553/file_arclog.txt create mode 100644 data/sample_backup/20090601/170553/file_database.txt create mode 100644 data/sample_backup/20090602/170553/backup.ini create mode 100644 data/sample_backup/20090603/170553/backup.ini create mode 100644 data/sample_backup/20090603/170553/file_arclog.txt create mode 100644 data/sample_backup/20090603/170553/file_database.txt create mode 100644 data/sample_backup/20090603/170553/file_srvlog.txt create mode 100644 data/sample_backup/pg_rman.ini create mode 100644 delete.c create mode 100644 dir.c create mode 100644 expected/backup_restore.out create mode 100644 expected/init.out create mode 100644 expected/option.out create mode 100644 expected/show_validate.out create mode 100644 init.c create mode 100644 parray.c create mode 100644 parray.h create mode 100644 pg_rman.c create mode 100644 pg_rman.h create mode 100644 pgsql_src/COPYRIGHT.pgsql_src create mode 100644 pgsql_src/pg_crc.c create mode 100644 pgsql_src/pg_ctl.c create mode 100644 pgut/pgut-port.c create mode 100644 pgut/pgut-port.h create mode 100644 pgut/pgut.c create mode 100644 pgut/pgut.h create mode 100644 restore.c create mode 100644 show.c create mode 100644 sql/backup.sql create mode 100644 sql/backup_restore.sh create mode 100644 sql/backup_restore.sql create mode 100644 sql/init.sql create mode 100644 sql/option.sh create mode 100644 sql/option.sql create mode 100644 sql/show_validate.sql create mode 100644 util.c create mode 100644 validate.c create mode 100644 xlog.c diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 00000000..17f79264 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,26 @@ +Copyright (c) 2008-2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION + (NTT) nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..b3c40792 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +PROGRAM = pg_rman +SRCS = \ + backup.c \ + catalog.c \ + data.c \ + delete.c \ + dir.c \ + init.c \ + parray.c \ + pg_rman.c \ + restore.c \ + show.c \ + util.c \ + validate.c \ + xlog.c \ + pgsql_src/pg_ctl.c \ + pgsql_src/pg_crc.c \ + pgut/pgut.c \ + pgut/pgut-port.c +OBJS = $(SRCS:.c=.o) +# pg_crc.c and are copied from PostgreSQL source tree. + +# XXX for debug, add -g and disable optimization +PG_CPPFLAGS = -I$(libpq_srcdir) +PG_LIBS = $(libpq_pgport) + +REGRESS = option init show_validate backup_restore + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_rman +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +$(OBJS): pg_rman.h diff --git a/backup.c b/backup.c new file mode 100644 index 00000000..14339afa --- /dev/null +++ b/backup.c @@ -0,0 +1,1110 @@ +/*------------------------------------------------------------------------- + * + * backup.c: backup DB cluster, archived WAL, serverlog. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include +#include +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "pgut/pgut-port.h" + +#define TIMEOUT_ARCHIVE 10 /* wait 10 sec until WAL archive complete */ + +static bool in_backup = false; /* TODO: more robust logic */ + +/* + * Backup routines + */ +static void backup_cleanup(bool fatal, void *userdata); +static void delete_old_files(const char *root, parray *files, int keep_files, int keep_days, bool is_arclog); +static void backup_files(const char *from_root, const char *to_root, + parray *files, parray *prev_files, XLogRecPtr *lsn, bool compress_data); +static parray *do_backup_database(parray *backup_list, bool smooth_checkpoint); +static parray *do_backup_arclog(parray *backup_list); +static parray *do_backup_srvlog(parray *backup_list); +static void confirm_compatibility(void); +static void confirm_block_size(const char *name, int blcksz); +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_stop_backup(pgBackup *backup); +static void pg_switch_xlog(pgBackup *backup); +static void get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn); + +static void delete_arclog_link(void); +static void delete_online_wal_backup(void); + +static bool fileExists(const char *path); + +/* + * Take a backup of database. + */ +static parray * +do_backup_database(parray *backup_list, bool smooth_checkpoint) +{ + int i; + parray *files; + parray *prev_files = NULL; /* file list of previous database backup */ + FILE *fp; + char path[MAXPGPATH]; + char label[1024]; + XLogRecPtr *lsn = NULL; + + if (!HAVE_DATABASE(¤t)) + return NULL; + + elog(INFO, _("database backup start")); + + /* initialize size summary */ + current.total_data_bytes = 0; + current.read_data_bytes = 0; + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_rman", lengthof(label)); + pg_start_backup(label, smooth_checkpoint, ¤t); + pgut_atexit_push(backup_cleanup, NULL); + + /* + * list directories and symbolic links with the physical path to make + * mkdirs.sh + * Sort in order of path. + * omit $PGDATA. + */ + files = parray_new(); + dir_list_file(files, pgdata, NULL, false, false); + + if (!check) + { + pgBackupGetPath(¤t, path, lengthof(path), MKDIRS_SH_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open make directory script \"%s\": %s"), + path, strerror(errno)); + dir_print_mkdirs_sh(fp, files, pgdata); + fclose(fp); + if (chmod(path, DIR_PERMISSION) == -1) + elog(ERROR_SYSTEM, _("can't change mode of \"%s\": %s"), path, + strerror(errno)); + } + + /* clear directory list */ + parray_walk(files, pgFileFree); + parray_free(files); + files = NULL; + + /* + * To take incremental backup, the file list of the last completed database + * backup is needed. + */ + if (current.backup_mode < BACKUP_MODE_FULL) + { + char prev_file_txt[MAXPGPATH]; + pgBackup *prev_backup; + + /* find last completed database backup */ + prev_backup = catalog_get_last_data_backup(backup_list); + if (prev_backup == NULL) + { + elog(INFO, _("no previous full backup, do a full backup instead")); + current.backup_mode = BACKUP_MODE_FULL; + } + else + { + pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt), + DATABASE_FILE_LIST); + prev_files = dir_read_file_list(pgdata, prev_file_txt); + + /* + * we back up only a page updated later than LSN of preview backup. + */ + lsn = &prev_backup->start_lsn; + elog(LOG, _("backup only the page that there was of the update from LSN(%X/%08X).\n"), + lsn->xlogid, lsn->xrecoff); + } + } + + /* list files with the logical path. omit $PGDATA */ + files = parray_new(); + dir_list_file(files, pgdata, pgdata_exclude, true, false); + + /* mark files as 'datafile' which are under base/global/pg_tblspc */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *relative; + char *fname; + + /* data file must be a regular file */ + if (!S_ISREG(file->mode)) + continue; + + /* data files are under base/global/pg_tblspc */ + relative = file->path + strlen(pgdata) + 1; + if (!path_is_prefix_of_path("base", relative) && + !path_is_prefix_of_path("global", relative) && + !path_is_prefix_of_path("pg_tblspc", relative)) + continue; + + /* name of data file start with digit */ + fname = last_dir_separator(relative); + if (fname == NULL) + fname = relative; + else + fname++; + if (!isdigit(fname[0])) + continue; + + file->is_datafile = true; + } + + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + backup_files(pgdata, path, files, prev_files, lsn, current.compress_data); + + /* notify end of backup */ + pg_stop_backup(¤t); + pgut_atexit_pop(backup_cleanup, NULL); + + /* create file list */ + if (!check) + { + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_FILE_LIST); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open file list \"%s\": %s"), path, + strerror(errno)); + dir_print_file_list(fp, files, pgdata); + fclose(fp); + } + + /* print sumarry of size of backup mode files */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (!S_ISREG(file->mode)) + continue; + current.total_data_bytes += file->size; + current.read_data_bytes += file->read_size; + if (file->write_size != BYTES_INVALID) + current.write_bytes += file->write_size; + } + + if (verbose) + { + printf(_("database backup completed(read: " INT64_FORMAT " write: " INT64_FORMAT ")\n"), + current.read_data_bytes, current.write_bytes); + printf(_("========================================\n")); + } + + return files; +} + +/* + * backup archived WAL incrementally. + */ +static parray * +do_backup_arclog(parray *backup_list) +{ + int i; + parray *files; + parray *prev_files = NULL; /* file list of previous database backup */ + FILE *fp; + char path[MAXPGPATH]; + char timeline_dir[MAXPGPATH]; + char prev_file_txt[MAXPGPATH]; + pgBackup *prev_backup; + int64 arclog_write_bytes = 0; + char last_wal[MAXPGPATH]; + + if (!HAVE_ARCLOG(¤t)) + return NULL; + + if (verbose) + { + printf(_("========================================\n")); + printf(_("archived WAL backup start\n")); + } + + /* initialize size summary */ + current.read_arclog_bytes = 0; + + /* switch xlog if database is not backed up */ + if (current.stop_lsn.xrecoff == 0) + pg_switch_xlog(¤t); + + /* + * To take incremental backup, the file list of the last completed database + * backup is needed. + */ + prev_backup = catalog_get_last_arclog_backup(backup_list); + if (verbose && prev_backup == NULL) + printf(_("no previous full backup, do a full backup instead\n")); + + if (prev_backup) + { + pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt), + ARCLOG_FILE_LIST); + prev_files = dir_read_file_list(arclog_path, prev_file_txt); + } + + /* list files with the logical path. omit ARCLOG_PATH */ + files = parray_new(); + dir_list_file(files, arclog_path, NULL, true, false); + + /* remove WALs archived after pg_stop_backup()/pg_switch_xlog() */ + xlog_fname(last_wal, lengthof(last_wal), current.tli, ¤t.stop_lsn); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *fname; + if ((fname = last_dir_separator(file->path))) + fname++; + else + fname = file->path; + + /* to backup backup hisotry files, compare tli/lsn portion only */ + if (strncmp(fname, last_wal, 24) > 0) + { + parray_remove(files, i); + i--; + } + } + + pgBackupGetPath(¤t, path, lengthof(path), ARCLOG_DIR); + backup_files(arclog_path, path, files, prev_files, NULL, + current.compress_data); + + /* create file list */ + if (!check) + { + pgBackupGetPath(¤t, path, lengthof(path), ARCLOG_FILE_LIST); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open file list \"%s\": %s"), path, + strerror(errno)); + dir_print_file_list(fp, files, arclog_path); + fclose(fp); + } + + /* print sumarry of size of backup files */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (!S_ISREG(file->mode)) + continue; + current.read_arclog_bytes += file->read_size; + if (file->write_size != BYTES_INVALID) + { + current.write_bytes += file->write_size; + arclog_write_bytes += file->write_size; + } + } + + /* + * Backup timeline history files to special directory. + * We do this after create file list, because copy_file() update + * pgFile->write_size to actual size. + */ + snprintf(timeline_dir, lengthof(timeline_dir), "%s/%s", backup_path, + TIMELINE_HISTORY_DIR); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (!S_ISREG(file->mode)) + continue; + if (strstr(file->path, ".history") == + file->path + strlen(file->path) - strlen(".history")) + { + elog(LOG, _("(timeline history) %s"), file->path); + copy_file(arclog_path, timeline_dir, file, NO_COMPRESSION); + } + } + + if (verbose) + { + printf(_("archived WAL backup completed(read: " INT64_FORMAT " write: " INT64_FORMAT ")\n"), + current.read_arclog_bytes, arclog_write_bytes); + printf(_("========================================\n")); + } + + return files; +} + +/* + * Take a backup of serverlog. + */ +static parray * +do_backup_srvlog(parray *backup_list) +{ + int i; + parray *files; + parray *prev_files = NULL; /* file list of previous database backup */ + FILE *fp; + char path[MAXPGPATH]; + char prev_file_txt[MAXPGPATH]; + pgBackup *prev_backup; + int64 srvlog_write_bytes = 0; + + if (!current.with_serverlog) + return NULL; + + if (verbose) + { + printf(_("========================================\n")); + printf(_("serverlog backup start\n")); + } + + /* initialize size summary */ + current.read_srvlog_bytes = 0; + + /* + * To take incremental backup, the file list of the last completed database + * backup is needed. + */ + prev_backup = catalog_get_last_srvlog_backup(backup_list); + if (verbose && prev_backup == NULL) + printf(_("no previous full backup, do a full backup instead\n")); + + if (prev_backup) + { + pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt), + SRVLOG_FILE_LIST); + prev_files = dir_read_file_list(srvlog_path, prev_file_txt); + } + + /* list files with the logical path. omit SRVLOG_PATH */ + files = parray_new(); + dir_list_file(files, srvlog_path, NULL, true, false); + + pgBackupGetPath(¤t, path, lengthof(path), SRVLOG_DIR); + backup_files(srvlog_path, path, files, prev_files, NULL, false); + + /* create file list */ + if (!check) + { + pgBackupGetPath(¤t, path, lengthof(path), SRVLOG_FILE_LIST); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open file list \"%s\": %s"), path, + strerror(errno)); + dir_print_file_list(fp, files, srvlog_path); + fclose(fp); + } + + /* print sumarry of size of backup mode files */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (!S_ISREG(file->mode)) + continue; + current.read_srvlog_bytes += file->read_size; + if (file->write_size != BYTES_INVALID) + { + current.write_bytes += file->write_size; + srvlog_write_bytes += file->write_size; + } + } + + if (verbose) + { + printf(_("serverlog backup completed(read: " INT64_FORMAT " write: " INT64_FORMAT ")\n"), + current.read_srvlog_bytes, srvlog_write_bytes); + printf(_("========================================\n")); + } + + return files; +} + +int +do_backup(bool smooth_checkpoint, + int keep_arclog_files, + int keep_arclog_days, + int keep_srvlog_files, + int keep_srvlog_days, + int keep_data_generations, + int keep_data_days) +{ + int ret; + parray *backup_list; + parray *files_database; + parray *files_arclog; + parray *files_srvlog; + + /* PGDATA and BACKUP_MODE are always required */ + if (pgdata == NULL) + elog(ERROR_ARGS, _("required parameter not specified: PGDATA (-D, --pgdata)")); + + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR_ARGS, _("required parameter not specified: BACKUP_MODE (-b, --backup-mode)")); + + /* ARCLOG_PATH is requried only when backup archive WAL */ + if (HAVE_ARCLOG(¤t) && arclog_path == NULL) + elog(ERROR_ARGS, _("required parameter not specified: ARCLOG_PATH (-A, --arclog-path)")); + + /* SRVLOG_PATH is required only when backup serverlog */ + if (current.with_serverlog && srvlog_path == NULL) + elog(ERROR_ARGS, _("required parameter not specified: SRVLOG_PATH (-S, --srvlog-path)")); + +#ifndef HAVE_LIBZ + if (current->compress_data) + { + elog(WARNING, _("requested compression not available in this: installation -- archive will be uncompressed")); + current->compress_data = false; + } +#endif + + /* confirm data block size and xlog block size are compatible */ + confirm_compatibility(); + + /* setup cleanup callback function */ + in_backup = true; + + /* show configuration acutally used */ + if (verbose) + { + printf(_("========================================\n")); + printf(_("backup start\n")); + printf(_("----------------------------------------\n")); + pgBackupWriteConfigSection(stderr, ¤t); + printf(_("----------------------------------------\n")); + } + + /* get exclusive lock of backup catalog */ + ret = catalog_lock(); + if (ret == -1) + elog(ERROR_SYSTEM, _("can't lock backup catalog.")); + else if (ret == 1) + elog(ERROR_ALREADY_RUNNING, + _("another pg_rman is running, skip this backup.")); + + /* initialize backup result */ + current.status = BACKUP_STATUS_RUNNING; + current.tli = 0; /* get from result of pg_start_backup() */ + current.start_lsn.xlogid = 0; + current.start_lsn.xrecoff = 0; + current.stop_lsn.xlogid = 0; + current.stop_lsn.xrecoff = 0; + current.start_time = time(NULL); + current.end_time = (time_t) 0; + current.total_data_bytes = BYTES_INVALID; + current.read_data_bytes = BYTES_INVALID; + current.read_arclog_bytes = BYTES_INVALID; + current.read_srvlog_bytes = BYTES_INVALID; + current.write_bytes = 0; /* write_bytes is valid always */ + current.block_size = BLCKSZ; + current.wal_block_size = XLOG_BLCKSZ; + + /* create backup directory and backup.ini */ + if (!check) + { + if (pgBackupCreateDir(¤t)) + elog(ERROR_SYSTEM, _("can't create backup directory.")); + pgBackupWriteIni(¤t); + } + if (verbose) + printf(_("backup destination is initialized.\n")); + + /* get list of backups already taken */ + backup_list = catalog_get_backup_list(NULL); + + /* backup data */ + files_database = do_backup_database(backup_list, smooth_checkpoint); + + /* backup archived WAL */ + files_arclog = do_backup_arclog(backup_list); + + /* backup serverlog */ + files_srvlog = do_backup_srvlog(backup_list); + + /* update backup status to DONE */ + current.end_time = time(NULL); + current.status = BACKUP_STATUS_DONE; + if (!check) + pgBackupWriteIni(¤t); + + if (verbose) + { + if (TOTAL_READ_SIZE(¤t) == 0) + printf(_("nothing to backup\n")); + else + printf(_("all backup completed(read: " INT64_FORMAT " write: " + INT64_FORMAT ")\n"), + TOTAL_READ_SIZE(¤t), current.write_bytes); + printf(_("========================================\n")); + } + + /* + * Delete old files (archived WAL and serverlog) after update of status. + */ + if (HAVE_ARCLOG(¤t)) + delete_old_files(arclog_path, files_arclog, keep_arclog_files, + keep_arclog_days, true); + if (current.with_serverlog) + delete_old_files(srvlog_path, files_srvlog, keep_srvlog_files, + keep_srvlog_days, false); + + /* Delete old backup files after all backup operation. */ + pgBackupDelete(keep_data_generations, keep_data_days); + + /* Cleanup backup mode file list */ + if (files_database) + parray_walk(files_database, pgFileFree); + parray_free(files_database); + if (files_arclog) + parray_walk(files_arclog, pgFileFree); + parray_free(files_arclog); + if (files_srvlog) + parray_walk(files_srvlog, pgFileFree); + parray_free(files_srvlog); + + /* + * If this backup is full backup, delete backup of online WAL. + * Note that sereverlog files which were backed up during first restoration + * don't be delete. + * Also delete symbolic link in the archive directory. + */ + if (current.backup_mode == BACKUP_MODE_FULL) + { + delete_online_wal_backup(); + delete_arclog_link(); + } + + /* release catalog lock */ + catalog_unlock(); + + return 0; +} + +/* + * Is the server compatible with this program? + */ +static void +confirm_compatibility(void) +{ + int server_version; + + reconnect(); + + /* confirm server version */ + server_version = PQserverVersion(connection); + if (server_version / 100 != PG_VERSION_NUM / 100) + elog(ERROR_PG_INCOMPATIBLE, + _("version mismatch. server is %d and %s is %d."), + server_version, PROGRAM_NAME, PG_VERSION_NUM); + + /* confirm block_size (BLCKSZ) and wal_block_size (XLOG_BLCKSZ) */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + disconnect(); +} + +static void +confirm_block_size(const char *name, int blcksz) +{ + PGresult *res; + char *endp; + int block_size; + + res = execute( + "SELECT setting from pg_settings where name = $1", + 1, &name); + if (PQntuples(res) != 1 || PQnfields(res) != 1) + elog(ERROR_PG_COMMAND, _("can't get %s: %s"), + name, PQerrorMessage(connection)); + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); + PQclear(res); + if ((endp && *endp) || block_size != blcksz) + elog(ERROR_PG_INCOMPATIBLE, + _("%s(%d) is not compatible(%d expected)"), + name, block_size, blcksz); +} + +/* + * Notify start of backup to PostgreSQL server. + */ +static void +pg_start_backup(const char *label, bool smooth, pgBackup *backup) +{ + PGresult *res; + const char *params[2]; + + params[0] = label; + + reconnect(); + if (PQserverVersion(connection) >= 80400) + { + params[1] = smooth ? "false" : "true"; + res = execute("SELECT * from pg_xlogfile_name_offset(pg_start_backup($1, $2))", 2, params); + } + else + { + /* v8.3 always uses smooth checkpoint */ + if (!smooth && PQserverVersion(connection) >= 80300) + command("CHECKPOINT", 0, NULL); + res = execute("SELECT * from pg_xlogfile_name_offset(pg_start_backup($1))", 1, params); + } + if (backup != NULL) + get_lsn(res, &backup->tli, &backup->start_lsn); + PQclear(res); + disconnect(); +} + +static void +wait_for_archive(pgBackup *backup, const char *sql) +{ + PGresult *res; + char ready_path[MAXPGPATH]; + int try_count; + + reconnect(); + res = execute(sql, 0, NULL); + if (backup != NULL) + { + get_lsn(res, &backup->tli, &backup->stop_lsn); + elog(LOG, _("%s(): tli=%X lsn=%X/%08X"), __FUNCTION__, backup->tli, + backup->stop_lsn.xlogid, backup->stop_lsn.xrecoff); + } + + /* get filename from the result of pg_xlogfile_name_offset() */ + snprintf(ready_path, lengthof(ready_path), + "%s/pg_xlog/archive_status/%s.ready", pgdata, PQgetvalue(res, 0, 0)); + elog(LOG, "%s() wait for %s", __FUNCTION__, ready_path); + + PQclear(res); + disconnect(); + + /* wait until switched WAL is archived */ + try_count = 0; + while (fileExists(ready_path)) + { + sleep(1); + if (interrupted) + elog(ERROR_INTERRUPTED, + _("interrupted during waiting for WAL archiving")); + try_count++; + if (try_count > TIMEOUT_ARCHIVE) + elog(ERROR_ARCHIVE_FAILED, + _("switched WAL could not be archived in %d seconds"), + TIMEOUT_ARCHIVE); + } + elog(LOG, "%s() .ready deleted in %d try", __FUNCTION__, try_count); +} + +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(pgBackup *backup) +{ + wait_for_archive(backup, + "SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup())"); +} + +/* + * Force switch to a new transaction log file and update backup->tli. + */ +static void +pg_switch_xlog(pgBackup *backup) +{ + wait_for_archive(backup, + "SELECT * FROM pg_xlogfile_name_offset(pg_switch_xlog())"); +} + +/* + * Get TimeLineID and LSN from result of pg_xlogfile_name_offset(). + */ +static void +get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn) +{ + uint32 off_upper; + + if (res == NULL || PQntuples(res) != 1 || PQnfields(res) != 2) + elog(ERROR_PG_COMMAND, + _("result of pg_xlogfile_name_offset() is invalid: %s"), + PQerrorMessage(connection)); + + /* get TimeLineID, LSN from result of pg_stop_backup() */ + if (sscanf(PQgetvalue(res, 0, 0), "%08X%08X%08X", + timeline, &lsn->xlogid, &off_upper) != 3 || + sscanf(PQgetvalue(res, 0, 1), "%u", &lsn->xrecoff) != 1) + { + elog(ERROR_PG_COMMAND, + _("result of pg_xlogfile_name_offset() is invalid: %s"), + PQerrorMessage(connection)); + } + + elog(LOG, "%s():%s %s", + __FUNCTION__, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 1)); + lsn->xrecoff += off_upper << 24; +} + +/* + * Return true if the path is a existing regular file. + */ +static bool +fileExists(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1 && errno == ENOENT) + return false; + else if (!S_ISREG(buf.st_mode)) + return false; + else + return true; +} + +/* + * Notify end of backup to server when "backup_label" is in the root directory + * of the DB cluster. + * Also update backup status to ERROR when the backup is not finished. + */ +static void +backup_cleanup(bool fatal, void *userdata) +{ + char path[MAXPGPATH]; + + if (!in_backup) + return; + + /* If backup_label exist in $PGDATA, notify stop of backup to PostgreSQL */ + snprintf(path, lengthof(path), "%s/backup_label", pgdata); + make_native_path(path); + if (fileExists(path)) + { + if (verbose) + printf(_("backup_label exists, stop backup\n")); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } + + /* + * Update status of backup.ini to ERROR. + * end_time != 0 means backup finished + */ + if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) + { + if (verbose) + printf(_("backup is running, update its status to ERROR\n")); + current.end_time = time(NULL); + current.status = BACKUP_STATUS_ERROR; + pgBackupWriteIni(¤t); + } +} + +/* take incremental backup. */ +static void +backup_files(const char *from_root, const char *to_root, parray *files, + parray *prev_files, XLogRecPtr *lsn, bool compress_data) +{ + int i; + struct timeval tv; + + /* sort pathname ascending */ + parray_qsort(files, pgFileComparePath); + + gettimeofday(&tv, NULL); + + /* backup a file or create a directory */ + for (i = 0; i < parray_num(files); i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(files, i); + + /* check for interrupt */ + if (interrupted) + elog(ERROR_INTERRUPTED, _("interrupted during backup")); + + /* print progress in verbose mode */ + if (verbose) + printf(_("(%d/%lu) %s "), i + 1, (unsigned long) parray_num(files), + file->path + strlen(from_root) + 1); + + /* stat file to get file type, size and modify timestamp */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + if (verbose) + printf(_("deleted\n")); + continue; + } + else + { + if (verbose) + printf("\n"); + elog(ERROR_SYSTEM, + _("can't stat backup mode. \"%s\": %s"), + file->path, strerror(errno)); + } + } + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(buf.st_mode)) + { + char dirpath[MAXPGPATH]; + snprintf(dirpath, lengthof(dirpath), "%s/%s", to_root, + file->path + strlen(from_root) + 1); + if (!check) + dir_create_dir(dirpath, DIR_PERMISSION); + if (verbose) + printf(_("directory\n")); + } + else if (S_ISREG(buf.st_mode)) + { + /* skip files which have not been modified since last backup */ + if (prev_files) + { + pgFile **p; + pgFile *prev_file = NULL; + + p = (pgFile **) parray_bsearch(prev_files, file, + pgFileComparePath); + if (p) + prev_file = *p; + + if (prev_file && prev_file->mtime >= file->mtime) + { + /* record as skipped file in file_xxx.txt */ + file->write_size = BYTES_INVALID; + if (verbose) + printf(_("skip\n")); + continue; + } + } + + /* + * We will wait until the next second of mtime so that backup + * file should contain all modifications at the clock of mtime. + * timer resolution of ext3 file system is one second. + */ + if (tv.tv_sec <= file->mtime) + { + /* update time and recheck */ + gettimeofday(&tv, NULL); + while (tv.tv_sec <= file->mtime) + { + usleep(1000000 - tv.tv_usec); + gettimeofday(&tv, NULL); + } + } + + /* copy the file into backup */ + if (file->is_datafile) + { + backup_data_file(from_root, to_root, file, lsn, compress_data); + if (file->write_size == 0 && file->read_size > 0) + { + /* record as skipped file in file_xxx.txt */ + file->write_size = BYTES_INVALID; + if (verbose) + printf(_("skip\n")); + continue; + } + } + else + copy_file(from_root, to_root, file, + compress_data ? COMPRESSION : NO_COMPRESSION); + + if (verbose) + { + /* print compression rate */ + if (file->write_size != file->size) + printf(_("compressed %lu (%.2f%% of %lu)\n"), + (unsigned long) file->write_size, + 100.0 * file->write_size / file->size, + (unsigned long) file->size); + else + printf(_("copied %lu\n"), (unsigned long) file->write_size); + } + + } + else + { + if (verbose) + printf(_(" unexpected file type %d\n"), buf.st_mode); + } + } +} + +/* + * Delete files modified before than KEEP_xxx_DAYS or more than KEEP_xxx_FILES + * of newer files exist. + */ +static void +delete_old_files(const char *root, parray *files, int keep_files, int keep_days, bool is_arclog) +{ + int i; + int j; + int file_num = 0; + time_t days_threashold = + current.start_time - (keep_days * 60 * 60 * 24); + + if (verbose) + { + char files_str[100]; + char days_str[100]; + + if (keep_files == KEEP_INFINITE) + strncpy(files_str, "INFINITE", lengthof(files_str)); + else + snprintf(files_str, lengthof(files_str), "%d", keep_files); + + if (keep_days == KEEP_INFINITE) + strncpy(days_str, "INFINITE", lengthof(days_str)); + else + snprintf(days_str, lengthof(days_str), "%d", keep_days); + + printf(_("delete old files from \"%s\" (files=%s, days=%s)\n"), + root, files_str, days_str); + } + + /* delete files which satisfy both condition */ + if (keep_files == KEEP_INFINITE || keep_days == KEEP_INFINITE) + { + elog(LOG, "%s() infinite", __FUNCTION__); + return; + } + + parray_qsort(files, pgFileCompareMtime); + for (i = parray_num(files) - 1; i >= 0; i--) + { + pgFile *file = (pgFile *) parray_get(files, i); + + elog(LOG, "%s() %s", __FUNCTION__, file->path); + /* Delete complete WAL only. */ + if (is_arclog && !xlog_is_complete_wal(file)) + { + elog(LOG, "%s() not complete WAL", __FUNCTION__); + continue; + } + + file_num++; + + /* + * If the mtime of the file is older than the threashold and there are + * enough number of files newer than the files, delete the file. + */ + if (file->mtime >= days_threashold) + { + elog(LOG, "%s() %lu is not older than %lu", __FUNCTION__, + file->mtime, days_threashold); + continue; + } + elog(LOG, "%s() %lu is older than %lu", __FUNCTION__, + file->mtime, days_threashold); + + if (file_num <= keep_files) + { + elog(LOG, "%s() newer files are only %d", __FUNCTION__, file_num); + continue; + } + + /* Now we found a file should be deleted. */ + if (verbose) + printf(_("delete \"%s\"\n"), file->path + strlen(root) + 1); + + /* delete corresponding backup history file if any */ + file = (pgFile *) parray_remove(files, i); + for (j = parray_num(files) - 1; j >= 0; j--) + { + pgFile *file2 = (pgFile *)parray_get(files, j); + if (strstr(file2->path, file->path) == file2->path) + { + file2 = (pgFile *)parray_remove(files, j); + if (verbose) + printf(_("delete \"%s\"\n"), + file2->path + strlen(root) + 1); + if (!check) + pgFileDelete(file2); + pgFileFree(file2); + } + } + if (!check) + pgFileDelete(file); + pgFileFree(file); + } +} + +static void +delete_online_wal_backup(void) +{ + int i; + parray *files = parray_new(); + char work_path[MAXPGPATH]; + + if (verbose) + { + printf(_("========================================\n")); + printf(_("delete online WAL backup\n")); + } + + snprintf(work_path, lengthof(work_path), "%s/%s/%s", backup_path, + RESTORE_WORK_DIR, PG_XLOG_DIR); + /* don't delete root dir */ + dir_list_file(files, work_path, NULL, true, false); + if (parray_num(files) == 0) + { + parray_free(files); + return; + } + + parray_qsort(files, pgFileComparePathDesc); /* delete from leaf */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (verbose) + printf(_("delete \"%s\"\n"), file->path); + if (!check) + pgFileDelete(file); + } + + parray_walk(files, pgFileFree); + parray_free(files); +} + +/* + * Remove symbolic links point ardchived WAL in backup catalog. + */ +static void +delete_arclog_link(void) +{ + int i; + parray *files = parray_new(); + + if (verbose) + { + printf(_("========================================\n")); + printf(_("delete symbolic link in archive directory\n")); + } + + dir_list_file(files, arclog_path, NULL, false, false); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISLNK(file->mode)) + continue; + + if (verbose) + printf(_("delete \"%s\"\n"), file->path); + + if (!check && remove(file->path) == -1) + elog(ERROR_SYSTEM, _("can't remove link \"%s\": %s"), file->path, + strerror(errno)); + } + + parray_walk(files, pgFileFree); + parray_free(files); +} diff --git a/catalog.c b/catalog.c new file mode 100644 index 00000000..40e327b0 --- /dev/null +++ b/catalog.c @@ -0,0 +1,593 @@ +/*------------------------------------------------------------------------- + * + * catalog.c: backup catalog opration + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pgut/pgut-port.h" + +static pgBackup *catalog_read_ini(const char *path); + +#define BOOL_TO_STR(val) ((val) ? "true" : "false") + +static int lock_fd = -1; + +/* + * Lock of the catalog with pg_rman.ini file and return 0. + * If the lock is held by another one, return 1 immediately. + */ +int +catalog_lock(void) +{ + int ret; + char id_path[MAXPGPATH]; + + snprintf(id_path, lengthof(id_path), "%s/%s", backup_path, + PG_RMAN_INI_FILE); + lock_fd = open(id_path, O_RDWR); + if (lock_fd == -1) + elog(errno == ENOENT ? ERROR_CORRUPTED : ERROR_SYSTEM, + _("can't open file \"%s\": %s"), id_path, strerror(errno)); + + ret = flock(lock_fd, LOCK_EX | LOCK_NB); /* non-blocking */ + if (ret == -1) + { + if (errno == EWOULDBLOCK) + { + close(lock_fd); + return 1; + } + else + { + int errno_tmp = errno; + close(lock_fd); + elog(ERROR_SYSTEM, _("can't lock file \"%s\": %s"), id_path, + strerror(errno_tmp)); + } + } + + return 0; +} + +/* + * Release catalog lock. + */ +void +catalog_unlock(void) +{ + close(lock_fd); + lock_fd = -1; +} + +/* + * Create a pgBackup which taken at timestamp. + * If no backup matches, return NULL. + */ +pgBackup * +catalog_get_backup(time_t timestamp) +{ + pgBackup tmp; + char ini_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, ini_path, lengthof(ini_path), BACKUP_INI_FILE); + + return catalog_read_ini(ini_path); +} + +static bool +IsDir(const char *dirpath, const DIR *dir, const struct dirent *ent) +{ +#if defined(DT_DIR) + return ent->d_type == DT_DIR; +#elif defined(_finddata_t) + return (dir->dd_dta.attrib & FILE_ATTRIBUTE_DIRECTORY) != 0; +#else + char path[MAXPGPATH]; + DWORD attr; + + /* dirent.d_type does not exists */ + strlcpy(path, dirpath, MAXPGPATH); + strlcat(path, "/", MAXPGPATH); + strlcat(path, ent->d_name, MAXPGPATH); + attr = GetFileAttributes(path); + return attr != (DWORD) -1 && (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; +#endif +} + +/* + * Create list fo backups started between begin and end from backup catalog. + * If range was NULL, all of backup are listed. + * The list is sorted in order of descending start time. + */ +parray * +catalog_get_backup_list(const pgBackupRange *range) +{ + const pgBackupRange range_all = { 0, 0 }; + DIR *date_dir = NULL; + struct dirent *date_ent = NULL; + DIR *time_dir = NULL; + struct dirent *time_ent = NULL; + char date_path[MAXPGPATH]; + parray *backups = NULL; + pgBackup *backup = NULL; + struct tm *tm; + char begin_date[100]; + char begin_time[100]; + char end_date[100]; + char end_time[100]; + + if (range == NULL) + range = &range_all; + + /* make date/time string */ + tm = localtime(&range->begin); + strftime(begin_date, lengthof(begin_date), "%Y%m%d", tm); + strftime(begin_time, lengthof(begin_time), "%H%M%S", tm); + tm = localtime(&range->end); + strftime(end_date, lengthof(end_date), "%Y%m%d", tm); + strftime(end_time, lengthof(end_time), "%H%M%S", tm); + + /* open backup root directory */ + date_dir = opendir(backup_path); + if (date_dir == NULL) + { + elog(WARNING, _("can't open directory \"%s\": %s"), backup_path, + strerror(errno)); + goto err_proc; + } + + /* scan date/time directories and list backups in the range */ + backups = parray_new(); + for (; (date_ent = readdir(date_dir)) != NULL; errno = 0) + { + /* skip not-directory entries and hidden entries */ + if (!IsDir(backup_path, date_dir, date_ent) || date_ent->d_name[0] == '.') + continue; + + /* skip online WAL & serverlog backup directory */ + if (strcmp(date_ent->d_name, RESTORE_WORK_DIR) == 0) + continue; + + /* If the date is out of range, skip it. */ + if (pgBackupRangeIsValid(range) && + (strcmp(begin_date, date_ent->d_name) > 0 || + strcmp(end_date, date_ent->d_name) < 0)) + continue; + + /* open subdirectory (date directory) and search time directory */ + snprintf(date_path, MAXPGPATH, "%s/%s", backup_path, date_ent->d_name); + time_dir = opendir(date_path); + if (time_dir == NULL) + { + elog(WARNING, _("can't open directory \"%s\": %s"), + date_ent->d_name, strerror(errno)); + goto err_proc; + } + for (; (time_ent = readdir(time_dir)) != NULL; errno = 0) + { + char ini_path[MAXPGPATH]; + + /* skip not-directry and hidden directories */ + if (!IsDir(date_path, date_dir, time_ent) || time_ent->d_name[0] == '.') + continue; + + /* If the time is out of range, skip it. */ + if (pgBackupRangeIsValid(range) && + (strcmp(begin_time, time_ent->d_name) > 0 || + strcmp(end_time, time_ent->d_name) < 0)) + continue; + + /* read backup information from backup.ini */ + snprintf(ini_path, MAXPGPATH, "%s/%s/%s", date_path, + time_ent->d_name, BACKUP_INI_FILE); + backup = catalog_read_ini(ini_path); + /* ignore corrupted backup */ + if (backup) + { + parray_append(backups, backup); + backup = NULL; + } + } + if (errno && errno != ENOENT) + { + elog(WARNING, _("can't read date directory \"%s\": %s"), + date_ent->d_name, strerror(errno)); + goto err_proc; + } + closedir(time_dir); + time_dir = NULL; + } + if (errno) + { + elog(WARNING, _("can't read backup root directory \"%s\": %s"), + backup_path, strerror(errno)); + goto err_proc; + } + + closedir(date_dir); + date_dir = NULL; + + parray_qsort(backups, pgBackupCompareIdDesc); + + return backups; + +err_proc: + if (time_dir) + closedir(time_dir); + if (date_dir) + closedir(date_dir); + if (backup) + pgBackupFree(backup); + if (backups) + parray_walk(backups, pgBackupFree); + parray_free(backups); + return NULL; +} + +/* + * Find the last completed database backup from the backup list. + */ +pgBackup * +catalog_get_last_data_backup(parray *backup_list) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, i); + + /* we need completed database backup */ + if (backup->status == BACKUP_STATUS_OK && HAVE_DATABASE(backup)) + return backup; + } + + return NULL; +} + +/* + * Find the last completed archived WAL backup from the backup list. + */ +pgBackup * +catalog_get_last_arclog_backup(parray *backup_list) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, i); + + /* we need completed archived WAL backup */ + if (backup->status == BACKUP_STATUS_OK && HAVE_ARCLOG(backup)) + return backup; + } + + return NULL; +} + +/* + * Find the last completed serverlog backup from the backup list. + */ +pgBackup * +catalog_get_last_srvlog_backup(parray *backup_list) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, i); + + /* we need completed serverlog backup */ + if (backup->status == BACKUP_STATUS_OK && backup->with_serverlog) + return backup; + } + + return NULL; +} + +/* create backup directory in $BACKUP_PATH */ +int +pgBackupCreateDir(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char *subdirs[] = { DATABASE_DIR, ARCLOG_DIR, SRVLOG_DIR, NULL }; + + pgBackupGetPath(backup, path, lengthof(path), NULL); + dir_create_dir(path, DIR_PERMISSION); + + /* create directories for actual backup files */ + for (i = 0; subdirs[i]; i++) + { + pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + dir_create_dir(path, DIR_PERMISSION); + } + + return 0; +} + +/* + * Write configuration section of backup.in to stream "out". + */ +void +pgBackupWriteConfigSection(FILE *out, pgBackup *backup) +{ + static const char *modes[] = { "", "ARCHIVE", "INCREMENTAL", "FULL"}; + + fprintf(out, "# configuration\n"); + + fprintf(out, "BACKUP_MODE=%s\n", modes[backup->backup_mode]); + fprintf(out, "WITH_SERVERLOG=%s\n", BOOL_TO_STR(backup->with_serverlog)); + fprintf(out, "COMPRESS_DATA=%s\n", BOOL_TO_STR(backup->compress_data)); +} + +/* + * Write result section of backup.in to stream "out". + */ +void +pgBackupWriteResultSection(FILE *out, pgBackup *backup) +{ + char timestamp[20]; + + fprintf(out, "# result\n"); + fprintf(out, "TIMELINEID=%d\n", backup->tli); + fprintf(out, "START_LSN=%x/%08x\n", backup->start_lsn.xlogid, + backup->start_lsn.xrecoff); + fprintf(out, "STOP_LSN=%x/%08x\n", backup->stop_lsn.xlogid, + backup->stop_lsn.xrecoff); + time2iso(timestamp, lengthof(timestamp), backup->start_time); + fprintf(out, "START_TIME='%s'\n", timestamp); + time2iso(timestamp, lengthof(timestamp), backup->end_time); + fprintf(out, "END_TIME='%s'\n", timestamp); + fprintf(out, "TOTAL_DATA_BYTES=" INT64_FORMAT "\n", backup->total_data_bytes); + fprintf(out, "READ_DATA_BYTES=" INT64_FORMAT "\n", backup->read_data_bytes); + fprintf(out, "READ_ARCLOG_BYTES=" INT64_FORMAT "\n", backup->read_arclog_bytes); + fprintf(out, "READ_SRVLOG_BYTES=" INT64_FORMAT "\n", backup->read_srvlog_bytes); + fprintf(out, "WRITE_BYTES=" INT64_FORMAT "\n", backup->write_bytes); + + fprintf(out, "BLOCK_SIZE=%u\n", backup->block_size); + fprintf(out, "XLOG_BLOCK_SIZE=%u\n", backup->wal_block_size); + + fprintf(out, "STATUS=%s\n", status2str(backup->status)); +} + +/* create backup.ini */ +void +pgBackupWriteIni(pgBackup *backup) +{ + FILE *fp = NULL; + char ini_path[MAXPGPATH]; + + pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_INI_FILE); + fp = fopen(ini_path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open INI file \"%s\": %s"), ini_path, + strerror(errno)); + + /* configuration section */ + pgBackupWriteConfigSection(fp, backup); + + /* result section */ + pgBackupWriteResultSection(fp, backup); + + fclose(fp); +} + +/* + * Read backup.ini and create pgBackup. + * - Comment starts with ';'. + * - Do not care section. + */ +static pgBackup * +catalog_read_ini(const char *path) +{ + pgBackup *backup; + char *backup_mode = NULL; + char *start_lsn = NULL; + char *stop_lsn = NULL; + char *status = NULL; + int i; + + pgut_option options[] = + { + { 's', 0, "backup-mode" , NULL, SOURCE_ENV }, + { 'b', 0, "with-serverlog" , NULL, SOURCE_ENV }, + { 'b', 0, "compress-data" , NULL, SOURCE_ENV }, + { 'u', 0, "timelineid" , NULL, SOURCE_ENV }, + { 's', 0, "start-lsn" , NULL, SOURCE_ENV }, + { 's', 0, "stop-lsn" , NULL, SOURCE_ENV }, + { 't', 0, "start-time" , NULL, SOURCE_ENV }, + { 't', 0, "end-time" , NULL, SOURCE_ENV }, + { 'I', 0, "total-data-bytes" , NULL, SOURCE_ENV }, + { 'I', 0, "read-data-bytes" , NULL, SOURCE_ENV }, + { 'I', 0, "read-arclog-bytes" , NULL, SOURCE_ENV }, + { 'I', 0, "read-srvlog-bytes" , NULL, SOURCE_ENV }, + { 'I', 0, "write-bytes" , NULL, SOURCE_ENV }, + { 'u', 0, "block-size" , NULL, SOURCE_ENV }, + { 'u', 0, "xlog-block-size" , NULL, SOURCE_ENV }, + { 's', 0, "status" , NULL, SOURCE_ENV }, + { 0 } + }; + + backup = (pgBackup *) pgut_malloc(sizeof(*backup)); + catalog_init_config(backup); + + i = 0; + options[i++].var = &backup_mode; + options[i++].var = &backup->with_serverlog; + options[i++].var = &backup->compress_data; + options[i++].var = &backup->tli; + options[i++].var = &start_lsn; + options[i++].var = &stop_lsn; + options[i++].var = &backup->start_time; + options[i++].var = &backup->end_time; + options[i++].var = &backup->total_data_bytes; + options[i++].var = &backup->read_data_bytes; + options[i++].var = &backup->read_arclog_bytes; + options[i++].var = &backup->read_srvlog_bytes; + options[i++].var = &backup->write_bytes; + options[i++].var = &backup->block_size; + options[i++].var = &backup->wal_block_size; + options[i++].var = &status; + Assert(i == lengthof(options) - 1); + + pgut_readopt(path, options, ERROR_CORRUPTED); + + if (backup_mode) + { + backup->backup_mode = parse_backup_mode(backup_mode, WARNING); + free(backup_mode); + } + + if (start_lsn) + { + XLogRecPtr lsn; + if (sscanf(start_lsn, "%X/%X", &lsn.xlogid, &lsn.xrecoff) == 2) + backup->start_lsn = lsn; + else + elog(WARNING, _("invalid START_LSN \"%s\""), start_lsn); + free(start_lsn); + } + + if (stop_lsn) + { + XLogRecPtr lsn; + if (sscanf(stop_lsn, "%X/%X", &lsn.xlogid, &lsn.xrecoff) == 2) + backup->stop_lsn = lsn; + else + elog(WARNING, _("invalid STOP_LSN \"%s\""), stop_lsn); + free(stop_lsn); + } + + if (status) + { + if (strcmp(status, "OK") == 0) + backup->status = BACKUP_STATUS_OK; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "ERROR") == 0) + backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "DELETING") == 0) + backup->status = BACKUP_STATUS_DELETING; + else if (strcmp(status, "DELETED") == 0) + backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DONE") == 0) + backup->status = BACKUP_STATUS_DONE; + else if (strcmp(status, "CORRUPT") == 0) + backup->status = BACKUP_STATUS_CORRUPT; + else + elog(WARNING, _("invalid STATUS \"%s\""), status); + free(status); + } + + return backup; +} + +BackupMode +parse_backup_mode(const char *value, int elevel) +{ + const char *v = value; + size_t len; + + while (IsSpace(*v)) { v++; } + len = strlen(v); + + if (len > 0 && pg_strncasecmp("full", v, len) == 0) + return BACKUP_MODE_FULL; + else if (len > 0 && pg_strncasecmp("incremental", v, len) == 0) + return BACKUP_MODE_INCREMENTAL; + else if (len > 0 && pg_strncasecmp("archive", v, len) == 0) + return BACKUP_MODE_ARCHIVE; + + elog(elevel, _("invalid backup-mode \"%s\""), value); + return BACKUP_MODE_INVALID; +} + +/* free pgBackup object */ +void +pgBackupFree(void *backup) +{ + free(backup); +} + +/* Compare two pgBackup with their ID (start time) in ascending order */ +int +pgBackupCompareId(const void *l, const void *r) +{ + pgBackup *lp = *(pgBackup **)l; + pgBackup *rp = *(pgBackup **)r; + + if (lp->start_time > rp->start_time) + return 1; + else if (lp->start_time < rp->start_time) + return -1; + else + return 0; +} + +/* Compare two pgBackup with their ID in descending order */ +int +pgBackupCompareIdDesc(const void *l, const void *r) +{ + return -pgBackupCompareId(l, r); +} + +/* + * Construct absolute path of the backup directory. + * If subdir is not NULL, it will be appended after the path. + */ +void +pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + char datetime[20]; + struct tm *tm; + + /* generate $BACKUP_PATH/date/time path */ + tm = localtime(&backup->start_time); + strftime(datetime, lengthof(datetime), "%Y%m%d/%H%M%S", tm); + if (subdir) + snprintf(path, len, "%s/%s/%s", backup_path, datetime, subdir); + else + snprintf(path, len, "%s/%s", backup_path, datetime); +} + +void +catalog_init_config(pgBackup *backup) +{ + backup->backup_mode = BACKUP_MODE_INVALID; + backup->with_serverlog = false; + backup->compress_data = false; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn.xlogid = 0; + backup->start_lsn.xrecoff = 0; + backup->stop_lsn.xlogid = 0; + backup->stop_lsn.xrecoff = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->total_data_bytes = BYTES_INVALID; + backup->read_data_bytes = BYTES_INVALID; + backup->read_arclog_bytes = BYTES_INVALID; + backup->read_srvlog_bytes = BYTES_INVALID; + backup->write_bytes = BYTES_INVALID; +} diff --git a/data.c b/data.c new file mode 100644 index 00000000..b848ee68 --- /dev/null +++ b/data.c @@ -0,0 +1,886 @@ +/*------------------------------------------------------------------------- + * + * data.c: compress / uncompress data pages + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "storage/block.h" +#include "storage/bufpage.h" + +#if PG_VERSION_NUM < 80300 +#define XLogRecPtrIsInvalid(r) ((r).xrecoff == 0) +#endif + +#ifdef HAVE_LIBZ +#include + +#define zlibOutSize 4096 +#define zlibInSize 4096 + +static int doDeflate(z_stream *zp, size_t in_size, size_t out_size, void *inbuf, + void *outbuf, FILE *in, FILE *out, pg_crc32 *crc, size_t *write_size, + int flash); +static int doInflate(z_stream *zp, size_t in_size, size_t out_size,void *inbuf, + void *outbuf, FILE *in, FILE *out, pg_crc32 *crc, size_t *read_size); + +static int +doDeflate(z_stream *zp, size_t in_size, size_t out_size, void *inbuf, + void *outbuf, FILE *in, FILE *out, pg_crc32 *crc, size_t *write_size, + int flash) +{ + int status; + + zp->next_in = inbuf; + zp->avail_in = in_size; + + /* compresses until an input buffer becomes empty. */ + do + { + if (interrupted) + elog(ERROR_INTERRUPTED, _("interrupted during deflate")); + + status = deflate(zp, flash); + + if (status == Z_STREAM_ERROR) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't compress data: %s"), zp->msg); + } + + if (fwrite(outbuf, 1, out_size - zp->avail_out, out) != + out_size - zp->avail_out) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write file: %s"), strerror(errno)); + } + + /* update CRC */ + COMP_CRC32(*crc, outbuf, out_size - zp->avail_out); + + *write_size += out_size - zp->avail_out; + + zp->next_out = outbuf; + zp->avail_out = out_size; + } while (zp->avail_in != 0); + + return status; +} + +static int +doInflate(z_stream *zp, size_t in_size, size_t out_size,void *inbuf, + void *outbuf, FILE *in, FILE *out, pg_crc32 *crc, size_t *read_size) +{ + int status = Z_OK; + + zp->next_out = outbuf; + zp->avail_out = out_size; + + /* decompresses until an output buffer becomes full. */ + for (;;) + { + if (interrupted) + elog(ERROR_INTERRUPTED, _("interrupted during inflate")); + + /* input buffer becomes empty, read it from a file. */ + if (zp->avail_in == 0) + { + size_t read_len; + + read_len = fread(inbuf, 1, in_size, in); + + if (read_len != in_size) + { + int errno_tmp = errno; + + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR_CORRUPTED, + _("can't read compress file: %s"), strerror(errno_tmp)); + } + + if (read_len == 0 && *read_size == 0) + return Z_STREAM_END; + } + + zp->next_in = inbuf; + zp->avail_in = read_len; + *read_size += read_len; + } + + /* decompresses input file data */ + status = inflate(zp, Z_NO_FLUSH); + + if (status == Z_STREAM_END) + { + if (feof(in)) + break; + /* not reached to EOF, read again */ + } + else if (status == Z_OK) + { + if (zp->avail_out == 0) + break; + /* more input needed to fill out_buf */ + } + else if (status != Z_OK) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't uncompress data: %s"), strerror(errno)); + } + } + + /* update CRC */ + COMP_CRC32(*crc, outbuf, out_size - zp->avail_out); + + return status; +} +#endif + +typedef union DataPage +{ + PageHeaderData header; + char data[BLCKSZ]; +} DataPage; + +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + uint16 hole_offset; /* number of bytes before "hole" */ + uint16 hole_length; /* number of bytes in "hole" */ +} BackupPageHeader; + +static bool +is_valid_header(const PageHeader page) +{ + const char *pagebytes; + int i; + + /* Check normal case */ + if (PageGetPageSize(page) == BLCKSZ && + PageGetPageLayoutVersion(page) == PG_PAGE_LAYOUT_VERSION && +#if PG_VERSION_NUM >= 80300 + (page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && +#endif + page->pd_lower >= SizeOfPageHeaderData && + page->pd_lower <= page->pd_upper && + page->pd_upper <= page->pd_special && + page->pd_special <= BLCKSZ && + page->pd_special == MAXALIGN(page->pd_special)) + return true; + + /* Check all-zeroes case */ + pagebytes = (char *) page; + for (i = 0; i < BLCKSZ; i++) + { + if (pagebytes[i] != 0) + return false; + } + return true; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. + * If lsn is not NULL, pages only which are modified after the lsn will be + * copied. + */ +void +backup_data_file(const char *from_root, const char *to_root, + pgFile *file, const XLogRecPtr *lsn, bool compress_data) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + BackupPageHeader header; + DataPage page; /* used as read buffer */ + BlockNumber blknum; + size_t read_len; + int errno_tmp; + pg_crc32 crc; +#ifdef HAVE_LIBZ + z_stream z; + char outbuf[zlibOutSize]; +#endif + + /* open backup mode file for read */ + in = fopen(file->path, "r"); + if (in == NULL) + { + /* meybe vanished, it's not error */ + if (errno == ENOENT) + return; + + elog(ERROR_SYSTEM, _("can't open backup mode file \"%s\": %s"), + file->path, strerror(errno)); + } + + /* open backup file for write */ + if (check) + snprintf(to_path, lengthof(to_path), "%s/tmp", backup_path); + else + snprintf(to_path, lengthof(to_path), "%s/%s", + to_root, file->path + strlen(from_root) + 1); + out = fopen(to_path, "w"); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR_SYSTEM, _("can't open backup file \"%s\": %s"), + to_path, strerror(errno_tmp)); + } + + INIT_CRC32(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + +#ifdef HAVE_LIBZ + if (compress_data) + { + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (deflateInit(&z, Z_DEFAULT_COMPRESSION) != Z_OK) + elog(ERROR_SYSTEM, _("can't initialize compression library: %s"), + z.msg); + + z.avail_in = 0; + z.next_out = (void *) outbuf; + z.avail_out = zlibOutSize; + } +#endif + + /* read each page and write the page excluding hole */ + for (blknum = 0; + (read_len = fread(&page, 1, sizeof(page), in)) == sizeof(page); + ++blknum) + { + int upper_offset; + int upper_length; + + header.block = blknum; + + /* + * If a invalid data page was found, fallback to simple copy to ensure + * all pages in the file don't have BackupPageHeader. + */ + if (!is_valid_header(&page.header) || + !XLogRecPtrIsInvalid(PageGetLSN(&page.header))) + { + elog(LOG, "%s fall back to simple copy", file->path); + fclose(in); + fclose(out); + file->is_datafile = false; + copy_file(from_root, to_root, file, + compress_data ? COMPRESSION : NO_COMPRESSION); + return; + } + + file->read_size += read_len; + + /* if the page has not been modified since last backup, skip it */ + if (lsn && !XLogRecPtrIsInvalid(PageGetLSN(&page.header)) && + XLByteLT(PageGetLSN(&page.header), *lsn)) + continue; + + header.hole_offset = page.header.pd_lower; + header.hole_length = page.header.pd_upper - page.header.pd_lower; + + upper_offset = header.hole_offset + header.hole_length; + upper_length = BLCKSZ - upper_offset; + +#ifdef HAVE_LIBZ + if (compress_data) + { + doDeflate(&z, sizeof(header), sizeof(outbuf), &header, outbuf, in, + out, &crc, &file->write_size, Z_NO_FLUSH); + doDeflate(&z, header.hole_offset, sizeof(outbuf), page.data, outbuf, + in, out, &crc, &file->write_size, Z_NO_FLUSH); + doDeflate(&z, upper_length, sizeof(outbuf), + page.data + upper_offset, outbuf, in, out, &crc, + &file->write_size, Z_NO_FLUSH); + } + else +#endif + { + /* write data page excluding hole */ + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header) || + fwrite(page.data, 1, header.hole_offset, out) != header.hole_offset || + fwrite(page.data + upper_offset, 1, upper_length, out) != upper_length) + { + int errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write at block %u of \"%s\": %s"), + blknum, to_path, strerror(errno_tmp)); + } + + /* update CRC */ + COMP_CRC32(crc, &header, sizeof(header)); + COMP_CRC32(crc, page.data, header.hole_offset); + COMP_CRC32(crc, page.data + upper_offset, upper_length); + + file->write_size += sizeof(header) + read_len - header.hole_length; + } + + } + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't read backup mode file \"%s\": %s"), + file->path, strerror(errno_tmp)); + } + + /* + * The odd size page at the tail is probably a page exactly written now, so + * write whole of it. + */ + if (read_len > 0) + { + /* + * If the odd size page is the 1st page, fallback to simple copy because + * the file is not a datafile. + * Otherwise treat the page as a datapage with no hole. + */ + if (blknum == 0) + file->is_datafile = false; + else + { + header.block = blknum; + header.hole_offset = 0; + header.hole_length = 0; + +#ifdef HAVE_LIBZ + if (compress_data) + { + doDeflate(&z, sizeof(header), sizeof(outbuf), &header, outbuf, + in, out, &crc, &file->write_size, Z_NO_FLUSH); + } + else +#endif + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + { + int errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, + _("can't write at block %u of \"%s\": %s"), + blknum, to_path, strerror(errno_tmp)); + } + COMP_CRC32(crc, &header, sizeof(header)); + file->write_size += sizeof(header); + } + } + + /* write odd size page image */ +#ifdef HAVE_LIBZ + if (compress_data) + { + doDeflate(&z, read_len, sizeof(outbuf), page.data, outbuf, in, out, + &crc, &file->write_size, Z_NO_FLUSH); + } + else +#endif + { + if (fwrite(page.data, 1, read_len, out) != read_len) + { + int errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write at block %u of \"%s\": %s"), + blknum, to_path, strerror(errno_tmp)); + } + + COMP_CRC32(crc, page.data, read_len); + file->write_size += read_len; + } + + file->read_size += read_len; + } + +#ifdef HAVE_LIBZ + if (compress_data) + { + if (file->read_size > 0) + { + while (doDeflate(&z, 0, sizeof(outbuf), NULL, outbuf, in, out, &crc, + &file->write_size, Z_FINISH) != Z_STREAM_END) + { + } + } + + if (deflateEnd(&z) != Z_OK) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't close compression stream: %s"), z.msg); + } + } + +#endif + /* update file permission */ + if (!check && chmod(to_path, FILE_PERMISSION) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't change mode of \"%s\": %s"), file->path, + strerror(errno_tmp)); + } + + fclose(in); + fclose(out); + + /* finish CRC calculation and store into pgFile */ + FIN_CRC32(crc); + file->crc = crc; + + /* Treat empty file as not-datafile */ + if (file->read_size == 0) + file->is_datafile = false; + + /* We do not backup if all pages skipped. */ + if (file->write_size == 0 && file->read_size > 0) + if (remove(to_path) == -1) + elog(ERROR_SYSTEM, _("can't remove file \"%s\": %s"), to_path, + strerror(errno)); + + /* remove $BACKUP_PATH/tmp created during check */ + if (check) + remove(to_path); +} + +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + */ +void +restore_data_file(const char *from_root, const char *to_root, pgFile *file, bool compress_data) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + BackupPageHeader header; + BlockNumber blknum; +#ifdef HAVE_LIBZ + z_stream z; + int status; + char inbuf[zlibInSize]; + pg_crc32 crc; + size_t read_size; +#endif + + /* If the file is not a datafile, copy it. */ + if (!file->is_datafile) + { + copy_file(from_root, to_root, file, + compress_data ? DECOMPRESSION : NO_COMPRESSION); + return; + } + + /* open backup mode file for read */ + in = fopen(file->path, "r"); + if (in == NULL) + { + elog(ERROR_SYSTEM, _("can't open backup file \"%s\": %s"), file->path, + strerror(errno)); + } + + /* open backup file for write */ + snprintf(to_path, lengthof(to_path), "%s/%s", to_root, + file->path + strlen(from_root) + 1); + out = fopen(to_path, "w"); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR_SYSTEM, _("can't open restore target file \"%s\": %s"), + to_path, strerror(errno_tmp)); + } + +#ifdef HAVE_LIBZ + if (compress_data) + { + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + z.next_in = Z_NULL; + z.avail_in = 0; + + if (inflateInit(&z) != Z_OK) + elog(ERROR_SYSTEM, _("can't initialize compression library: %s"), + z.msg); + INIT_CRC32(crc); + read_size = 0; + } +#endif + + for (blknum = 0; ;blknum++) + { + size_t read_len; + DataPage page; /* used as read buffer */ + int upper_offset; + int upper_length; + + /* read BackupPageHeader */ +#ifdef HAVE_LIBZ + if (compress_data) + { + status = doInflate(&z, sizeof(inbuf), sizeof(header), inbuf, + &header, in, out, &crc, &read_size); + if (status == Z_STREAM_END) + { + if (z.avail_out != sizeof(header)) + elog(ERROR_CORRUPTED, _("backup is broken header")); + break; + } + if (z.avail_out != 0) + elog(ERROR_SYSTEM, _("can't read block %u of \"%s\""), + blknum, file->path); + } + else +#endif + { + read_len = fread(&header, 1, sizeof(header), in); + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + { + elog(ERROR_CORRUPTED, + _("odd size page found at block %u of \"%s\""), + blknum, file->path); + } + else + { + elog(ERROR_SYSTEM, _("can't read block %u of \"%s\": %s"), + blknum, file->path, strerror(errno_tmp)); + } + } + } + + if (header.block < blknum || header.hole_offset > BLCKSZ || + (int) header.hole_offset + (int) header.hole_length > BLCKSZ) + { + elog(ERROR_CORRUPTED, _("backup is broken at block %u"), + blknum); + } + + upper_offset = header.hole_offset + header.hole_length; + upper_length = BLCKSZ - upper_offset; + + /* read lower/upper into page.data and restore hole */ + memset(page.data + header.hole_offset, 0, header.hole_length); +#ifdef HAVE_LIBZ + if (compress_data) + { + elog(LOG, "\n%s() %s %d %d", __FUNCTION__, file->path, header.hole_offset, upper_length); + if (header.hole_offset > 0) + { + doInflate(&z, sizeof(inbuf), header.hole_offset, inbuf, page.data, + in, out, &crc, &read_size); + if (z.avail_out != 0) + elog(ERROR_SYSTEM, _("can't read block %u of \"%s\""), + blknum, file->path); + } + + if (upper_length > 0) + { + doInflate(&z, sizeof(inbuf), upper_length, inbuf, + page.data + upper_offset, in, out, &crc, &read_size); + if (z.avail_out != 0) + elog(ERROR_SYSTEM, _("can't read block %u of \"%s\""), + blknum, file->path); + } + } + else +#endif + { + if (fread(page.data, 1, header.hole_offset, in) != header.hole_offset || + fread(page.data + upper_offset, 1, upper_length, in) != upper_length) + { + elog(ERROR_SYSTEM, _("can't read block %u of \"%s\": %s"), + blknum, file->path, strerror(errno)); + } + } + + /* by the incremental backup, we skip in a page without the update. */ + if (blknum != header.block) + { + if (fseek(out, (header.block - blknum) * BLCKSZ, SEEK_CUR) < 0) + elog(ERROR_SYSTEM, _("can't seek restore target file \"%s\": %s"), + to_path, strerror(errno)); + blknum = header.block; + } + + if (fwrite(page.data, 1, sizeof(page), out) != sizeof(page)) + { + elog(ERROR_SYSTEM, _("can't write block %u of \"%s\": %s"), + blknum, file->path, strerror(errno)); + } + + } + +#ifdef HAVE_LIBZ + if (compress_data && inflateEnd(&z) != Z_OK) + elog(ERROR_SYSTEM, _("can't close compression stream: %s"), z.msg); +#endif + + /* update file permission */ + if (chmod(to_path, file->mode) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't change mode of \"%s\": %s"), to_path, + strerror(errno_tmp)); + } + + fclose(in); + fclose(out); +} + +void +copy_file(const char *from_root, const char *to_root, pgFile *file, + CompressionMode mode) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[8192]; + struct stat st; + pg_crc32 crc; +#ifdef HAVE_LIBZ + z_stream z; + int status; + char outbuf[zlibOutSize]; + char inbuf[zlibInSize]; +#endif + + /* open backup mode file for read */ + in = fopen(file->path, "r"); + if (in == NULL) + { + /* meybe deleted, it's not error */ + if (errno == ENOENT) + return; + + elog(ERROR_SYSTEM, _("can't open source file \"%s\": %s"), file->path, + strerror(errno)); + } + + /* open backup file for write */ + if (check) + snprintf(to_path, lengthof(to_path), "%s/tmp", backup_path); + else + snprintf(to_path, lengthof(to_path), "%s/%s", to_root, + file->path + strlen(from_root) + 1); + out = fopen(to_path, "w"); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR_SYSTEM, _("can't open destination file \"%s\": %s"), + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + elog(ERROR_SYSTEM, _("can't stat \"%s\": %s"), file->path, + strerror(errno)); + } + + INIT_CRC32(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + +#ifdef HAVE_LIBZ + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (mode == COMPRESSION) + { + if (deflateInit(&z, Z_DEFAULT_COMPRESSION) != Z_OK) + elog(ERROR_SYSTEM, _("can't initialize compression library: %s"), + z.msg); + + z.avail_in = 0; + z.next_out = (void *) outbuf; + z.avail_out = zlibOutSize; + } + else if (mode == DECOMPRESSION) + { + z.next_in = Z_NULL; + z.avail_in = 0; + if (inflateInit(&z) != Z_OK) + elog(ERROR_SYSTEM, _("can't initialize compression library: %s"), + z.msg); + } +#endif + + /* copy content and calc CRC */ + for (;;) + { +#ifdef HAVE_LIBZ + if (mode == COMPRESSION) + { + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + doDeflate(&z, read_len, sizeof(outbuf), buf, outbuf, in, out, &crc, + &file->write_size, Z_NO_FLUSH); + file->read_size += sizeof(buf); + } + else if (mode == DECOMPRESSION) + { + status = doInflate(&z, sizeof(inbuf), sizeof(outbuf), inbuf, outbuf, + in, out, &crc, &file->read_size); + if (fwrite(outbuf, 1, sizeof(outbuf) - z.avail_out, out) != + sizeof(outbuf) - z.avail_out) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write to \"%s\": %s"), to_path, + strerror(errno_tmp)); + } + + file->write_size += sizeof(outbuf) - z.avail_out; + if (status == Z_STREAM_END) + break; + } + else +#endif + { + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write to \"%s\": %s"), to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32(crc, buf, read_len); + + file->write_size += sizeof(buf); + file->read_size += sizeof(buf); + } + } + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't read backup mode file \"%s\": %s"), + file->path, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (mode == COMPRESSION) + { + doDeflate(&z, read_len, sizeof(outbuf), buf, outbuf, in, out, &crc, + &file->write_size, Z_NO_FLUSH); + } + else +#endif + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't write to \"%s\": %s"), to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32(crc, buf, read_len); + + file->write_size += read_len; + } + + file->read_size += read_len; + } + +#ifdef HAVE_LIBZ + if (mode == COMPRESSION) + { + if (file->read_size > 0) + { + while (doDeflate(&z, 0, sizeof(outbuf), NULL, outbuf, in, out, &crc, + &file->write_size, Z_FINISH) != Z_STREAM_END) + { + } + } + + if (deflateEnd(&z) != Z_OK) + elog(ERROR_SYSTEM, _("can't close compression stream: %s"), z.msg); + } + else if (mode == DECOMPRESSION) + { + if (inflateEnd(&z) != Z_OK) + elog(ERROR_SYSTEM, _("can't close compression stream: %s"), z.msg); + } + +#endif + /* finish CRC calculation and store into pgFile */ + FIN_CRC32(crc); + file->crc = crc; + + /* update file permission */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR_SYSTEM, _("can't change mode of \"%s\": %s"), to_path, + strerror(errno_tmp)); + } + + fclose(in); + fclose(out); + + if (check) + remove(to_path); +} diff --git a/data/sample_backup/20090531/170553/backup.ini b/data/sample_backup/20090531/170553/backup.ini new file mode 100644 index 00000000..cb7d421c --- /dev/null +++ b/data/sample_backup/20090531/170553/backup.ini @@ -0,0 +1,18 @@ +# configuration +BACKUP_MODE=FULL +WITH_SERVERLOG=NO +COMPRESS_DATA=NO +# result +TIMELINEID=1 +START_LSN=0/0b40c800 +STOP_LSN=0/0b4c8020 +START_TIME='2009-05-31 17:05:53' +END_TIME='2009-05-31 17:09:13' +TOTAL_DATA_BYTES=1242102558 +READ_DATA_BYTES=1024 +READ_ARCLOG_BYTES=9223372036854775807 +READ_SRVLOG_BYTES=-1 +WRITE_BYTES=242102558 +BLOCK_SIZE=8192 +XLOG_BLOCK_SIZE=8192 +STATUS=DONE diff --git a/data/sample_backup/20090531/170553/database/PG_VERSION b/data/sample_backup/20090531/170553/database/PG_VERSION new file mode 100644 index 00000000..c9dc0490 --- /dev/null +++ b/data/sample_backup/20090531/170553/database/PG_VERSION @@ -0,0 +1 @@ +8.4 diff --git a/data/sample_backup/20090531/170553/file_arclog.txt b/data/sample_backup/20090531/170553/file_arclog.txt new file mode 100644 index 00000000..e69de29b diff --git a/data/sample_backup/20090531/170553/file_database.txt b/data/sample_backup/20090531/170553/file_database.txt new file mode 100644 index 00000000..d599b727 --- /dev/null +++ b/data/sample_backup/20090531/170553/file_database.txt @@ -0,0 +1 @@ +PG_VERSION f 4 4277607361 0600 2009-08-06 18:40:18 diff --git a/data/sample_backup/20090601/170553/backup.ini b/data/sample_backup/20090601/170553/backup.ini new file mode 100644 index 00000000..fd897a9d --- /dev/null +++ b/data/sample_backup/20090601/170553/backup.ini @@ -0,0 +1,18 @@ +# configuration +BACKUP_MODE=INCREMENTAL +WITH_SERVERLOG=NO +COMPRESS_DATA=NO +# result +TIMELINEID=1 +START_LSN=0/0b40c800 +STOP_LSN=0/0b4c8020 +START_TIME='2009-06-01 17:05:53' +END_TIME='2009-06-01 17:09:13' +TOTAL_DATA_BYTES=1242102558 +READ_DATA_BYTES=9223372036854775807 +READ_ARCLOG_BYTES=16777216 +READ_SRVLOG_BYTES=-1 +WRITE_BYTES=162372983 +BLOCK_SIZE=8192 +XLOG_BLOCK_SIZE=8192 +STATUS=DONE diff --git a/data/sample_backup/20090601/170553/database/PG_VERSION b/data/sample_backup/20090601/170553/database/PG_VERSION new file mode 100644 index 00000000..c9dc0490 --- /dev/null +++ b/data/sample_backup/20090601/170553/database/PG_VERSION @@ -0,0 +1 @@ +8.4 diff --git a/data/sample_backup/20090601/170553/file_arclog.txt b/data/sample_backup/20090601/170553/file_arclog.txt new file mode 100644 index 00000000..e69de29b diff --git a/data/sample_backup/20090601/170553/file_database.txt b/data/sample_backup/20090601/170553/file_database.txt new file mode 100644 index 00000000..8a26fa01 --- /dev/null +++ b/data/sample_backup/20090601/170553/file_database.txt @@ -0,0 +1 @@ +PG_VERSION f 4 0 0600 2009-08-06 18:40:18 diff --git a/data/sample_backup/20090602/170553/backup.ini b/data/sample_backup/20090602/170553/backup.ini new file mode 100644 index 00000000..1f443111 --- /dev/null +++ b/data/sample_backup/20090602/170553/backup.ini @@ -0,0 +1,18 @@ +# configuration +BACKUP_MODE=ARCHIVE +WITH_SERVERLOG=YES +COMPRESS_DATA=NO +# result +TIMELINEID=1 +START_LSN=0/0b40c800 +STOP_LSN=0/0b4c8020 +START_TIME='2009-06-02 17:05:03' +END_TIME='2009-06-02 17:05:03' +TOTAL_DATA_BYTES=-1 +READ_DATA_BYTES=-1 +READ_ARCLOG_BYTES=-1 +READ_SRVLOG_BYTES=4335423 +WRITE_BYTES=162372983 +BLOCK_SIZE=8192 +XLOG_BLOCK_SIZE=8192 +STATUS=DELETED diff --git a/data/sample_backup/20090603/170553/backup.ini b/data/sample_backup/20090603/170553/backup.ini new file mode 100644 index 00000000..bfe1c908 --- /dev/null +++ b/data/sample_backup/20090603/170553/backup.ini @@ -0,0 +1,18 @@ +# configuration +BACKUP_MODE=FULL +WITH_SERVERLOG=YES +COMPRESS_DATA=NO +# result +TIMELINEID=1 +START_LSN=0/0b40c800 +STOP_LSN=0/0b4c8020 +START_TIME='2009-06-03 17:05:53' +END_TIME='****-**-** **:**:**' +TOTAL_DATA_BYTES=-1 +READ_DATA_BYTES=-1 +READ_ARCLOG_BYTES=-1 +READ_SRVLOG_BYTES=-1 +WRITE_BYTES=-1 +BLOCK_SIZE=8192 +XLOG_BLOCK_SIZE=8192 +STATUS=RUNNING diff --git a/data/sample_backup/20090603/170553/file_arclog.txt b/data/sample_backup/20090603/170553/file_arclog.txt new file mode 100644 index 00000000..e69de29b diff --git a/data/sample_backup/20090603/170553/file_database.txt b/data/sample_backup/20090603/170553/file_database.txt new file mode 100644 index 00000000..e69de29b diff --git a/data/sample_backup/20090603/170553/file_srvlog.txt b/data/sample_backup/20090603/170553/file_srvlog.txt new file mode 100644 index 00000000..e69de29b diff --git a/data/sample_backup/pg_rman.ini b/data/sample_backup/pg_rman.ini new file mode 100644 index 00000000..e69de29b diff --git a/delete.c b/delete.c new file mode 100644 index 00000000..d6d9bdfa --- /dev/null +++ b/delete.c @@ -0,0 +1,235 @@ +/*------------------------------------------------------------------------- + * + * delete.c: delete backup files. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +static int pgBackupDeleteFiles(pgBackup *backup); + +int +do_delete(pgBackupRange *range) +{ + int i; + int ret; + parray *backup_list; + bool do_delete; + + /* DATE are always required */ + if (!pgBackupRangeIsValid(range)) + elog(ERROR_ARGS, _("required delete range option not specified: delete DATE")); + + /* get exclusive lock of backup catalog */ + ret = catalog_lock(); + if (ret == -1) + elog(ERROR_SYSTEM, _("can't lock backup catalog.")); + else if (ret == 1) + elog(ERROR_ALREADY_RUNNING, + _("another pg_rman is running, stop restore.")); + + /* get list of backups. */ + backup_list = catalog_get_backup_list(NULL); + + do_delete = false; + /* find delete target backup. */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *)parray_get(backup_list, i); + + /* delete backup and update status to DELETED */ + if (do_delete) + { + pgBackupDeleteFiles(backup); + continue; + } + + /* find latest full backup. */ + if (backup->backup_mode >= BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_OK && + backup->start_time <= range->begin) + do_delete = true; + } + + /* release catalog lock */ + catalog_unlock(); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + return 0; +} + +/* + * Delete backups that are older than KEEP_xxx_DAYS and have more generations + * than KEEP_xxx_FILES. + */ +void +pgBackupDelete(int keep_generations, int keep_days) +{ + int i; + parray *backup_list; + int backup_num; + time_t days_threashold = current.start_time - (keep_days * 60 * 60 * 24); + + if (verbose) + { + char generations_str[100]; + char days_str[100]; + + if (keep_generations == KEEP_INFINITE) + strncpy(generations_str, "INFINITE", + lengthof(generations_str)); + else + snprintf(generations_str, lengthof(generations_str), + "%d", keep_generations); + + if (keep_days == KEEP_INFINITE) + strncpy(days_str, "INFINITE", lengthof(days_str)); + else + snprintf(days_str, lengthof(days_str), "%d", keep_days); + + printf(_("delete old backups (generations=%s, days=%s)\n"), + generations_str, days_str); + } + + /* delete files which satisfy both condition */ + if (keep_generations == KEEP_INFINITE || keep_days == KEEP_INFINITE) + { + elog(LOG, "%s() infinite", __FUNCTION__); + return; + } + + /* get list of backups. */ + backup_list = catalog_get_backup_list(NULL); + + backup_num = 0; + /* find delete target backup. */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *)parray_get(backup_list, i); + + elog(LOG, "%s() %lu", __FUNCTION__, backup->start_time); + /* + * when validate full backup was found, we can delete the backup + * that is older than it + */ + if (backup->backup_mode >= BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_OK) + backup_num++; + + /* do not include the latest full backup in a count. */ + if (backup_num - 1 <= keep_generations) + { + elog(LOG, "%s() backup are only %d", __FUNCTION__, backup_num); + continue; + } + + /* + * If the start time of the backup is older than the threashold and + * there are enough generations of full backups, delete the backup. + */ + if (backup->start_time >= days_threashold) + { + elog(LOG, "%s() %lu is not older than %lu", __FUNCTION__, + backup->start_time, days_threashold); + continue; + } + + elog(LOG, "%s() %lu is older than %lu", __FUNCTION__, + backup->start_time, days_threashold); + + /* delete backup and update status to DELETED */ + pgBackupDeleteFiles(backup); + } + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Delete backup files of the backup and update the status of the backup to + * BACKUP_STATUS_DELETED. + */ +static int +pgBackupDeleteFiles(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char timestamp[20]; + parray *files; + + /* + * If the backup was deleted already, nothing to do and such situation + * is not error. + */ + if (backup->status == BACKUP_STATUS_DELETED) + return 0; + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + + elog(INFO, _("delete: %s"), timestamp); + + /* + * update STATUS to BACKUP_STATUS_DELETING in preparation for the case which + * the error occurs before deleting all backup files. + */ + if (!check) + { + backup->status = BACKUP_STATUS_DELETING; + pgBackupWriteIni(backup); + } + + /* list files to be deleted */ + files = parray_new(); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + dir_list_file(files, path, NULL, true, true); + pgBackupGetPath(backup, path, lengthof(path), ARCLOG_DIR); + dir_list_file(files, path, NULL, true, true); + pgBackupGetPath(backup, path, lengthof(path), SRVLOG_DIR); + dir_list_file(files, path, NULL, true, true); + + /* delete leaf node first */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + /* print progress */ + elog(LOG, _("delete file(%d/%lu) \"%s\"\n"), i + 1, + (unsigned long) parray_num(files), file->path); + + /* skip actual deletion in check mode */ + if (!check) + { + if (remove(file->path)) + { + elog(WARNING, _("can't remove \"%s\": %s"), file->path, + strerror(errno)); + parray_walk(files, pgFileFree); + parray_free(files); + return 1; + } + } + } + + /* + * After deleting all of the backup files, update STATUS to + * BACKUP_STATUS_DELETED. + */ + if (!check) + { + backup->status = BACKUP_STATUS_DELETED; + pgBackupWriteIni(backup); + } + + parray_walk(files, pgFileFree); + parray_free(files); + + return 0; +} diff --git a/dir.c b/dir.c new file mode 100644 index 00000000..65a9a5e2 --- /dev/null +++ b/dir.c @@ -0,0 +1,567 @@ +/*------------------------------------------------------------------------- + * + * dir.c: directory operation utility. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include +#include +#include +#include +#include +#include + +#include "pgut/pgut-port.h" + +/* directory exclusion list for backup mode listing */ +const char *pgdata_exclude[] = +{ + "pg_xlog", + "pg_stat_tmp", + "pgsql_tmp", + NULL, /* arclog_path will be set later */ + NULL, /* srvlog_path will be set later */ + NULL, /* centinel */ +}; + +static pgFile *pgFileNew(const char *path, bool omit_symlink); + +/* create directory, also create parent directories if necessary */ +int +dir_create_dir(const char *dir, mode_t mode) +{ + char copy[MAXPGPATH]; + char *parent; + + strncpy(copy, dir, MAXPGPATH); + parent= dirname(copy); + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode); + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST) /* already exist */ + return 0; + elog(ERROR_SYSTEM, _("can't create directory \"%s\": %s"), dir, + strerror(errno)); + } + + return 0; +} + +static pgFile * +pgFileNew(const char *path, bool omit_symlink) +{ + struct stat st; + pgFile *file; + + /* stat the file */ + if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + { + /* file not found is not an error case */ + if (errno == ENOENT) + return NULL; + elog(ERROR_SYSTEM, _("can't stat file \"%s\": %s"), path, + strerror(errno)); + } + + file = (pgFile *) pgut_malloc(offsetof(pgFile, path) + strlen(path) + 1); + + file->mtime = st.st_mtime; + file->size = st.st_size; + file->read_size = 0; + file->write_size = 0; + file->mode = st.st_mode; + file->crc = 0; + file->is_datafile = false; + file->linked = NULL; + strcpy(file->path, path); /* enough buffer size guaranteed */ + + return file; +} + +void +pgFileDump(pgFile *file, FILE *out) +{ + char mtime_str[100]; + + fprintf(out, "=================\n"); + if (file) + { + time2iso(mtime_str, 100, file->mtime); + fprintf(out, "mtime=%lu(%s)\n", file->mtime, mtime_str); + fprintf(out, "size=" UINT64_FORMAT "\n", (uint64)file->size); + fprintf(out, "read_size=" UINT64_FORMAT "\n", (uint64)file->read_size); + fprintf(out, "write_size=" UINT64_FORMAT "\n", (uint64)file->write_size); + fprintf(out, "mode=0%o\n", file->mode); + fprintf(out, "crc=%u\n", file->crc); + fprintf(out, "is_datafile=%s\n", file->is_datafile ? "true" : "false"); + fprintf(out, "linked=\"%s\"\n", file->linked ? file->linked : "nil"); + fprintf(out, "path=\"%s\"\n", file->path); + } + fprintf(out, "=================\n"); +} + +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +pgFileDelete(pgFile *file) +{ + if (S_ISDIR(file->mode)) + { + if (rmdir(file->path) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR_SYSTEM, _("can't remove directory \"%s\": %s"), + file->path, strerror(errno)); + } + return; + } + +delete_file: + if (remove(file->path) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR_SYSTEM, _("can't remove file \"%s\": %s"), file->path, + strerror(errno)); + } +} + +pg_crc32 +pgFileGetCRC(pgFile *file) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[1024]; + size_t len; + int errno_tmp; + + /* open file in binary read mode */ + fp = fopen(file->path, "r"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't open file \"%s\": %s"), + file->path, strerror(errno)); + + /* calc CRC of backup file */ + INIT_CRC32(crc); + while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + { + if (interrupted) + elog(ERROR_INTERRUPTED, _("interrupted during CRC calculation")); + COMP_CRC32(crc, buf, len); + } + errno_tmp = errno; + if (!feof(fp)) + elog(WARNING, _("can't read \"%s\": %s"), file->path, + strerror(errno_tmp)); + if (len > 0) + COMP_CRC32(crc, buf, len); + FIN_CRC32(crc); + + fclose(fp); + + return crc; +} + +void +pgFileFree(void *file) +{ + if (file == NULL) + return; + free(((pgFile *)file)->linked); + free(file); +} + +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileComparePath(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->path, f2p->path); +} + +/* Compare two pgFile with their path in descending order of ASCII code. */ +int +pgFileComparePathDesc(const void *f1, const void *f2) +{ + return -pgFileComparePath(f1, f2); +} + +/* Compare two pgFile with their modify timestamp. */ +int +pgFileCompareMtime(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + if (f1p->mtime > f2p->mtime) + return 1; + else if (f1p->mtime < f2p->mtime) + return -1; + else return 0; +} + +/* Compare two pgFile with their modify timestamp in descending order. */ +int +pgFileCompareMtimeDesc(const void *f1, const void *f2) +{ + return -pgFileCompareMtime(f1, f2); +} + +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * If the sub-directory name is in "exclude" list, the sub-directory itself is + * listed but the contents of the sub-directory is ignored. + * + * When omit_symlink is true, symbolic link is ignored and only file or + * directory llnked to will be listed. + */ +void +dir_list_file(parray *files, const char *root, const char *exclude[], bool omit_symlink, bool add_root) +{ + pgFile *file; + + file = pgFileNew(root, omit_symlink); + if (file == NULL) + return; + + if (add_root) + parray_append(files, file); + + /* chase symbolic link chain and find regular file or directory */ + while (S_ISLNK(file->mode)) + { + ssize_t len; + char linked[MAXPGPATH]; + + len = readlink(file->path, linked, sizeof(linked)); + if (len == -1) + { + elog(ERROR_SYSTEM, _("can't read link \"%s\": %s"), file->path, + strerror(errno)); + } + linked[len] = '\0'; + file->linked = pgut_strdup(linked); + + /* make absolute path to read linked file */ + if (linked[0] != '/') + { + char dname[MAXPGPATH]; + char *dnamep; + char absolute[MAXPGPATH]; + + strncpy(dname, file->path, lengthof(dname)); + dnamep = dirname(dname); + snprintf(absolute, lengthof(absolute), "%s/%s", dname, linked); + file = pgFileNew(absolute, omit_symlink); + } + else + file = pgFileNew(file->linked, omit_symlink); + + /* linked file is not found, stop chasing link chain */ + if (file == NULL) + return; + + parray_append(files, file); + } + + /* + * If the entry was a directory, add it to the list and add call this + * function recursivelly. + * If the directory name is in the exclude list, do not list the contents. + */ + while (S_ISDIR(file->mode)) + { + int i; + bool skip = false; + DIR *dir; + struct dirent *dent; + char *dirname; + + /* skip entry which matches exclude list */ + dirname = strrchr(file->path, '/'); + if (dirname == NULL) + dirname = file->path; + else + dirname++; + + /* + * If the item in the exclude list starts with '/', compare to th + * absolute path of the directory. Otherwise compare to the directory + * name portion. + */ + for (i = 0; exclude && exclude[i]; i++) + { + if (exclude[i][0] == '/') + { + if (strcmp(file->path, exclude[i]) == 0) + { + skip = true; + break; + } + } + else + { + if (strcmp(dirname, exclude[i]) == 0) + { + skip = true; + break; + } + } + } + if (skip) + break; + + /* open directory and list contents */ + dir = opendir(file->path); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* maybe the direcotry was removed */ + return; + } + elog(ERROR_SYSTEM, _("can't open directory \"%s\": %s"), + file->path, strerror(errno)); + } + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + snprintf(child, lengthof(child), "%s/%s", file->path, dent->d_name); + dir_list_file(files, child, exclude, omit_symlink, true); + } + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + closedir(dir); + elog(ERROR_SYSTEM, _("can't read directory \"%s\": %s"), + file->path, strerror(errno_tmp)); + } + closedir(dir); + + break; /* psuedo loop */ + } + + parray_qsort(files, pgFileComparePath); +} + +/* print mkdirs.sh */ +void +dir_print_mkdirs_sh(FILE *out, const parray *files, const char *root) +{ + int i; + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (S_ISDIR(file->mode)) + { + if (strstr(file->path, root) == file->path) + fprintf(out, "mkdir -m 700 -p %s\n", file->path + strlen(root) + + 1); + else + fprintf(out, "mkdir -m 700 -p %s\n", file->path); + } + } + + fprintf(out, "\n"); + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + if (S_ISLNK(file->mode)) + { + fprintf(out, "rm -f %s\n", file->path + strlen(root) + 1); + fprintf(out, "ln -s %s %s\n", file->linked, file->path + strlen(root) + 1); + } + } +} + +/* print file list */ +void +dir_print_file_list(FILE *out, const parray *files, const char *root) +{ + int i; + int root_len = 0; + + /* calculate length of root directory portion */ + if (root) + { + root_len = strlen(root); + if (root[root_len - 1] != '/') + root_len++; + } + + /* print each file in the list */ + for (i = 0; i < parray_num(files); i++) { + pgFile *file = (pgFile *)parray_get(files, i); + char *path = file->path; + char type; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = path + root_len; + + if (S_ISREG(file->mode) && file->is_datafile) + type = 'F'; + else if (S_ISREG(file->mode) && !file->is_datafile) + type = 'f'; + else if (S_ISDIR(file->mode)) + type = 'd'; + else if (S_ISLNK(file->mode)) + type = 'l'; + else + type = '?'; + + fprintf(out, "%s %c %lu %u 0%o", path, type, + (unsigned long) file->write_size, + file->crc, file->mode & (S_IRWXU | S_IRWXG | S_IRWXO)); + + if (S_ISLNK(file->mode)) + fprintf(out, " %s\n", file->linked); + else + { + char timestamp[20]; + time2iso(timestamp, 20, file->mtime); + fprintf(out, " %s\n", timestamp); + } + } +} + +/* + * Construct parray of pgFile from the file list. + * If root is not NULL, path will be absolute path. + */ +parray * +dir_read_file_list(const char *root, const char *file_txt) +{ + FILE *fp; + parray *files; + char buf[MAXPGPATH * 2]; + + fp = fopen(file_txt, "rt"); + if (fp == NULL) + elog(errno == ENOENT ? ERROR_CORRUPTED : ERROR_SYSTEM, + _("can't open \"%s\": %s"), file_txt, strerror(errno)); + + files = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char type; + unsigned long write_size; + pg_crc32 crc; + unsigned int mode; /* bit length of mode_t depends on platforms */ + struct tm tm; + pgFile *file; + + if (sscanf(buf, "%s %c %lu %u %o %d-%d-%d %d:%d:%d", + path, &type, &write_size, &crc, &mode, + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 11) + { + elog(ERROR_CORRUPTED, _("invalid format found in \"%s\""), + file_txt); + } + if (type != 'f' && type != 'F' && type != 'd' && type != 'l') + { + elog(ERROR_CORRUPTED, _("invalid type '%c' found in \"%s\""), + type, file_txt); + } + + file = (pgFile *) pgut_malloc(offsetof(pgFile, path) + + (root ? strlen(root) + 1 : 0) + strlen(path) + 1); + + tm.tm_year -= 1900; + tm.tm_mon -= 1; + file->mtime = mktime(&tm); + file->mode = mode | + ((type == 'f' || type == 'F') ? S_IFREG : + type == 'd' ? S_IFDIR : type == 'l' ? S_IFLNK : 0); + file->size = 0; + file->read_size = 0; + file->write_size = write_size; + file->crc = crc; + file->is_datafile = (type == 'F' ? true : false); + file->linked = NULL; + if (root) + sprintf(file->path, "%s/%s", root, path); + else + strcpy(file->path, path); + + parray_append(files, file); + } + + fclose(fp); + + /* file.txt is sorted, so this qsort is redundant */ + parray_qsort(files, pgFileComparePath); + + return files; +} + +/* copy contents of directory from_root into to_root */ +void +dir_copy_files(const char *from_root, const char *to_root) +{ + int i; + parray *files; + files = parray_new(); + + /* don't copy root directory */ + dir_list_file(files, from_root, NULL, true, false); + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + { + char to_path[MAXPGPATH]; + snprintf(to_path, lengthof(to_path), "%s/%s", to_root, + file->path + strlen(from_root) + 1); + if (verbose && !check) + printf(_("create directory \"%s\"\n"), + file->path + strlen(from_root) + 1); + if (!check) + dir_create_dir(to_path, DIR_PERMISSION); + continue; + } + else if(S_ISREG(file->mode)) + { + if (verbose && !check) + printf(_("copy \"%s\"\n"), + file->path + strlen(from_root) + 1); + if (!check) + copy_file(from_root, to_root, file, NO_COMPRESSION); + } + } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); +} diff --git a/expected/backup_restore.out b/expected/backup_restore.out new file mode 100644 index 00000000..33137b58 --- /dev/null +++ b/expected/backup_restore.out @@ -0,0 +1,40 @@ +\! sh sql/backup_restore.sh +Line style is ascii. +CREATE TABLESPACE +CREATE DATABASE +# of deleted backups +0 +full database backup +CHECKPOINT +incremental database backup +CHECKPOINT +CHECKPOINT +archived WAL and serverlog backup +stop DB during running pgbench +diff files in BACKUP_PATH/backup/pg_xlog +# of files in BACKUP_PATH/backup/srvlog +1 +diff files in BACKUP_PATH/backup/pg_xlog +# of files in BACKUP_PATH/backup/srvlog +2 +full database backup after recovery +CHECKPOINT +# of files in BACKUP_PATH/backup/pg_xlog +0 +# of files in BACKUP_PATH/backup/srvlog +2 +# of symbolic links in ARCLOG_PATH +0 +# of files in BACKUP_PATH/timeline_history +2 +# of recovery target option in recovery.conf +3 +# of deleted backups (show all) +4 +# of deleted backups +0 +delete backup +# of deleted backups +4 +# of deleted backups +9 diff --git a/expected/init.out b/expected/init.out new file mode 100644 index 00000000..5d73026d --- /dev/null +++ b/expected/init.out @@ -0,0 +1,14 @@ +\! rm -rf results/init_test +\! pg_rman init -B results/init_test --quiet;echo $? +WARNING: ARCLOG_PATH is not set because archive_command is empty +0 +\! find results/init_test | xargs ls -Fd | sort +results/init_test/ +results/init_test/backup/ +results/init_test/backup/pg_xlog/ +results/init_test/backup/srvlog/ +results/init_test/pg_rman.ini +results/init_test/timeline_history/ +\! pg_rman init -B results/init_test --quiet;echo $? +ERROR: backup catalog already exist. +2 diff --git a/expected/option.out b/expected/option.out new file mode 100644 index 00000000..fca36498 --- /dev/null +++ b/expected/option.out @@ -0,0 +1,80 @@ +\! sh sql/option.sh +pg_rman manage backup/recovery of PostgreSQL database. + +Usage: + pg_rman OPTION init + pg_rman OPTION backup + pg_rman OPTION restore + pg_rman OPTION show [DATE] + pg_rman OPTION show timeline [DATE] + pg_rman OPTION validate [DATE] + pg_rman OPTION delete DATE + +Common Options: + -D, --pgdata=PATH location of the database storage area + -A, --arclog-path=PATH location of archive WAL storage area + -S, --srvlog-path=PATH location of server log storage area + -B, --backup-path=PATH location of the backup storage area + -c, --check show what would have been done + +Backup options: + -b, --backup-mode=MODE full, incremental, or archive + -s, --with-serverlog also backup server log files + -Z, --compress-data compress data backup with zlib + -C, --smooth-checkpoint do smooth checkpoint before backup + --keep-data-generations=N keep GENERATION of full data backup + --keep-data-days=DAY keep enough data backup to recover to DAY days age + --keep-arclog-files=NUM keep NUM of archived WAL + --keep-arclog-days=DAY keep archived WAL modified in DAY days + --keep-srvlog-files=NUM keep NUM of serverlogs + --keep-srvlog-days=DAY keep serverlog modified in DAY days + +Restore options: + --recovery-target-time time stamp up to which recovery will proceed + --recovery-target-xid transaction ID up to which recovery will proceed + --recovery-target-inclusive whether we stop just after the recovery target + --recovery-target-timeline recovering into a particular timeline + +Catalog options: + -a, --show-all show deleted backup too + +Connection options: + -d, --dbname=DBNAME database to connect + -h, --host=HOSTNAME database server host or socket directory + -p, --port=PORT database server port + -U, --username=USERNAME user name to connect as + -w, --no-password never prompt for password + -W, --password force password prompt + +Generic options: + -q, --quiet don't write any messages + --debug debug mode + --help show this help, then exit + --version output version information, then exit + +Read the website for details. +Report bugs to . +pg_rman 1.1.0 +ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path) +ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode) +ERROR: required parameter not specified: ARCLOG_PATH (-A, --arclog-path) +ERROR: required parameter not specified: ARCLOG_PATH (-A, --arclog-path) +ERROR: required parameter not specified: ARCLOG_PATH (-A, --arclog-path) +ERROR: invalid backup-mode "bad" +ERROR: required delete range option not specified: delete DATE +INFO: validate: 2009-05-31 17:05:53 +INFO: validate: 2009-06-01 17:05:53 +WARNING: CRC of backup file "PG_VERSION" must be 0 but FEF71BC1 +WARNING: backup 2009-06-01 17:05:53 is corrupted +WARNING: syntax error in " = INFINITE". +ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode) +ERROR: invalid backup-mode "" +ERROR: required parameter not specified: ARCLOG_PATH (-A, --arclog-path) +ERROR: required parameter not specified: ARCLOG_PATH (-A, --arclog-path) +ERROR: invalid backup-mode "B" +ERROR: option -Z, --compress-data should be a boolean: 'FOO' +ERROR: option --keep-arclog-files should be a 32bit signed integer: 'YES' +ERROR: invalid option "TIMELINEID" +ERROR: invalid option "BACKUP_TARGETS" +ERROR: invalid backup-mode "F''\F" +ERROR: invalid backup-mode "ENV_PATH" diff --git a/expected/show_validate.out b/expected/show_validate.out new file mode 100644 index 00000000..955db304 --- /dev/null +++ b/expected/show_validate.out @@ -0,0 +1,50 @@ +-- test show command +\! rm -rf results/sample_backup +\! cp -rp data/sample_backup results/sample_backup +\! pg_rman show -B results/sample_backup +============================================================================ +Start Time Total Data WAL Log Backup Status +============================================================================ +2009-06-03 17:05:53 ---- ---- ---- ---- ---- ---- RUNNING +2009-06-01 17:05:53 3m ---- 9223PB 16MB ---- 162MB DONE +2009-05-31 17:05:53 3m 1242MB ---- 9223PB ---- 242MB DONE +\! pg_rman validate -B results/sample_backup 2009-05-31 17:05:53 --debug +INFO: validate: 2009-05-31 17:05:53 +LOG: database files... +LOG: (1/1) PG_VERSION +LOG: archive WAL files... +LOG: backup 2009-05-31 17:05:53 is valid +\! pg_rman validate -B results/sample_backup 2009-06-01 17:05:53 --debug +INFO: validate: 2009-06-01 17:05:53 +LOG: database files... +LOG: (1/1) PG_VERSION +WARNING: CRC of backup file "PG_VERSION" must be 0 but FEF71BC1 +LOG: archive WAL files... +WARNING: backup 2009-06-01 17:05:53 is corrupted +\! pg_rman show -a -B results/sample_backup +============================================================================ +Start Time Total Data WAL Log Backup Status +============================================================================ +2009-06-03 17:05:53 ---- ---- ---- ---- ---- ---- RUNNING +2009-06-02 17:05:03 0m ---- ---- ---- 4335kB 162MB DELETED +2009-06-01 17:05:53 3m ---- 9223PB 16MB ---- 162MB CORRUPT +2009-05-31 17:05:53 3m 1242MB ---- 9223PB ---- 242MB OK +\! pg_rman show 2009-06-01 17:05:53 -B results/sample_backup +# configuration +BACKUP_MODE=INCREMENTAL +WITH_SERVERLOG=false +COMPRESS_DATA=false +# result +TIMELINEID=1 +START_LSN=0/0b40c800 +STOP_LSN=0/0b4c8020 +START_TIME='2009-06-01 17:05:53' +END_TIME='2009-06-01 17:09:13' +TOTAL_DATA_BYTES=1242102558 +READ_DATA_BYTES=9223372036854775807 +READ_ARCLOG_BYTES=16777216 +READ_SRVLOG_BYTES=-1 +WRITE_BYTES=162372983 +BLOCK_SIZE=8192 +XLOG_BLOCK_SIZE=8192 +STATUS=CORRUPT diff --git a/init.c b/init.c new file mode 100644 index 00000000..ba3d4bee --- /dev/null +++ b/init.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * init.c: manage backup catalog. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include + +static void parse_postgresql_conf(const char *path, char **log_directory, + char **archive_command); + +/* + * Initialize backup catalog. + */ +int +do_init(void) +{ + char path[MAXPGPATH]; + char *log_directory = NULL; + char *archive_command = NULL; + FILE *fp; + + if (access(backup_path, F_OK) == 0) + elog(ERROR, _("backup catalog already exist.")); + + /* create backup catalog root directory */ + dir_create_dir(backup_path, DIR_PERMISSION); + + /* create directories for backup of online files */ + snprintf(path, lengthof(path), "%s/%s", backup_path, RESTORE_WORK_DIR); + dir_create_dir(path, DIR_PERMISSION); + snprintf(path, lengthof(path), "%s/%s/%s", backup_path, RESTORE_WORK_DIR, + PG_XLOG_DIR); + dir_create_dir(path, DIR_PERMISSION); + snprintf(path, lengthof(path), "%s/%s/%s", backup_path, RESTORE_WORK_DIR, + SRVLOG_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* create directory for timeline history files */ + snprintf(path, lengthof(path), "%s/%s", backup_path, TIMELINE_HISTORY_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* read postgresql.conf */ + if (pgdata) + { + snprintf(path, lengthof(path), "%s/%s", pgdata, "postgresql.conf"); + parse_postgresql_conf(path, &log_directory, &archive_command); + } + + /* create pg_rman.ini */ + snprintf(path, lengthof(path), "%s/%s", backup_path, PG_RMAN_INI_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR_SYSTEM, _("can't create pg_rman.ini: %s"), strerror(errno)); + + /* set ARCLOG_PATH refered with log_directory */ + if (arclog_path == NULL && archive_command && archive_command[0]) + { + char *command = pgut_strdup(archive_command); + char *begin; + char *end; + char *fname; + + /* example: 'cp "%p" /path/to/arclog/"%f"' */ + for (begin = command; *begin;) + { + begin = begin + strspn(begin, " \n\r\t\v"); + end = begin + strcspn(begin, " \n\r\t\v"); + *end = '\0'; + + if ((fname = strstr(begin, "%f")) != NULL) + { + while (strchr(" \n\r\t\v""'", *begin)) + begin++; + fname--; + while (fname > begin && strchr(" \n\r\t\v\"'/", fname[-1])) + fname--; + *fname = '\0'; + + if (is_absolute_path(begin)) + arclog_path = pgut_strdup(begin); + break; + } + + begin = end + 1; + } + + free(command); + } + if (arclog_path) + { + fprintf(fp, "ARCLOG_PATH='%s'\n", arclog_path); + elog(INFO, "ARCLOG_PATH is set to '%s'", arclog_path); + } + else if (archive_command && archive_command[0]) + elog(WARNING, "ARCLOG_PATH is not set because failed to parse archive_command '%s'", archive_command); + else + elog(WARNING, "ARCLOG_PATH is not set because archive_command is empty"); + + /* set SRVLOG_PATH refered with log_directory */ + if (srvlog_path == NULL) + { + if (log_directory) + { + if (is_absolute_path(log_directory)) + srvlog_path = pgut_strdup(log_directory); + else + { + srvlog_path = pgut_malloc(MAXPGPATH); + snprintf(srvlog_path, MAXPGPATH, "%s/%s", pgdata, log_directory); + } + } + else if (pgdata) + { + /* default: log_directory = 'pg_log' */ + srvlog_path = pgut_malloc(MAXPGPATH); + snprintf(srvlog_path, MAXPGPATH, "%s/%s", pgdata, "pg_log"); + } + } + if (srvlog_path) + { + fprintf(fp, "SRVLOG_PATH='%s'\n", srvlog_path); + elog(INFO, "SRVLOG_PATH is set to '%s'", srvlog_path); + } + + fprintf(fp, "\n"); + fclose(fp); + + free(archive_command); + free(log_directory); + + return 0; +} + +static void +parse_postgresql_conf(const char *path, + char **log_directory, + char **archive_command) +{ + pgut_option options[] = + { + { 's', 0, "log_directory" , NULL, SOURCE_ENV }, + { 's', 0, "archive_command" , NULL, SOURCE_ENV }, + { 0 } + }; + + options[0].var = log_directory; + options[1].var = archive_command; + + pgut_readopt(path, options, LOG); /* ignore unknown options */ +} diff --git a/parray.c b/parray.c new file mode 100644 index 00000000..35d8b669 --- /dev/null +++ b/parray.c @@ -0,0 +1,184 @@ +/*------------------------------------------------------------------------- + * + * parray.c: pointer array collection. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +/* members of struct parray are hidden from client. */ +struct parray +{ + void **data; /* poiter array, expanded if necessary */ + size_t alloced; /* number of elements allocated */ + size_t used; /* number of elements in use */ +}; + +/* + * Create new parray object. + * Never returns NULL. + */ +parray * +parray_new(void) +{ + parray *a = pgut_malloc(sizeof(parray)); + + a->data = NULL; + a->used = 0; + a->alloced = 0; + + parray_expand(a, 1024); + + return a; +} + +/* + * Expand array pointed by data to newsize. + * Elements in expanded area are initialized to NULL. + * Note: never returns NULL. + */ +void +parray_expand(parray *array, size_t newsize) +{ + void **p; + + /* already allocated */ + if (newsize <= array->alloced) + return; + + p = pgut_realloc(array->data, sizeof(void *) * newsize); + + /* initialize expanded area to NULL */ + memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); + + array->alloced = newsize; + array->data = p; + + return; +} + +void +parray_free(parray *array) +{ + if (array == NULL) + return; + free(array->data); + free(array); +} + +void +parray_append(parray *array, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + array->data[array->used++] = elem; +} + +void +parray_insert(parray *array, size_t index, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + memmove(array->data + index + 1, array->data + index, + (array->alloced - index - 1) * sizeof(void *)); + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; + else + array->used++; +} + +/* + * Concatinate two parray. + * parray_concat() appends the copy of the content of src to the end of dest. + */ +parray * +parray_concat(parray *dest, const parray *src) +{ + /* expand head array */ + parray_expand(dest, dest->used + src->used); + + /* copy content of src after content of dest */ + memcpy(dest->data + dest->used * sizeof(void *), src->data, + src->used * sizeof(void *)); + dest->used += parray_num(src); + + return dest; +} + +void +parray_set(parray *array, size_t index, void *elem) +{ + if (index > array->alloced - 1) + parray_expand(array, index + 1); + + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; +} + +void * +parray_get(const parray *array, size_t index) +{ + if (index > array->alloced - 1) + return NULL; + return array->data[index]; +} + +void * +parray_remove(parray *array, size_t index) +{ + void *val; + + /* removing unused element */ + if (index > array->used) + return NULL; + + val = array->data[index]; + + /* Do not move if the last element was removed. */ + if (index < array->alloced - 1) + memmove(array->data + index, array->data + index + 1, + (array->alloced - index - 1) * sizeof(void *)); + + /* adjust used count */ + array->used--; + + return val; +} + +size_t +parray_num(const parray *array) +{ + return array->used; +} + +void +parray_qsort(parray *array, int(*compare)(const void *, const void *)) +{ + qsort(array->data, array->used, sizeof(void *), compare); +} + +void +parray_walk(parray *array, void (*action)(void *)) +{ + int i; + for (i = 0; i < array->used; i++) + action(array->data[i]); +} + +void * +parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + return bsearch(&key, array->data, array->used, sizeof(void *), compare); +} + diff --git a/parray.h b/parray.h new file mode 100644 index 00000000..169cb194 --- /dev/null +++ b/parray.h @@ -0,0 +1,34 @@ +/*------------------------------------------------------------------------- + * + * parray.h: pointer array collection. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#ifndef PARRAY_H +#define PARRAY_H + +/* + * "parray" hold pointers to objects in a linear memory area. + * Client use "parray *" to access parray object. + */ +typedef struct parray parray; + +extern parray *parray_new(void); +extern void parray_expand(parray *array, size_t newnum); +extern void parray_free(parray *array); +extern void parray_append(parray *array, void *val); +extern void parray_insert(parray *array, size_t index, void *val); +extern parray *parray_concat(parray *head, const parray *tail); +extern void parray_set(parray *array, size_t index, void *val); +extern void *parray_get(const parray *array, size_t index); +extern void *parray_remove(parray *array, size_t index); +extern size_t parray_num(const parray *array); +extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); +extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern void parray_walk(parray *array, void (*action)(void *)); + +#endif /* PARRAY_H */ + diff --git a/pg_rman.c b/pg_rman.c new file mode 100644 index 00000000..ecfa3f18 --- /dev/null +++ b/pg_rman.c @@ -0,0 +1,301 @@ +/*------------------------------------------------------------------------- + * + * pg_rman.c: Backup/Recovery manager for PostgreSQL. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "pg_rman.h" + +#include +#include +#include + +const char *PROGRAM_VERSION = "1.1.1"; +const char *PROGRAM_URL = "https://code.google.com/p/pg-rman/"; +const char *PROGRAM_EMAIL = "https://code.google.com/p/pg-rman/issues/list"; + +/* path configuration */ +char *backup_path; +char *pgdata; +char *arclog_path; +char *srvlog_path; + +/* common configuration */ +bool verbose = false; +bool check = false; + +/* directory configuration */ +pgBackup current; + +/* backup configuration */ +static bool smooth_checkpoint; +static int keep_arclog_files = KEEP_INFINITE; +static int keep_arclog_days = KEEP_INFINITE; +static int keep_srvlog_files = KEEP_INFINITE; +static int keep_srvlog_days = KEEP_INFINITE; +static int keep_data_generations = KEEP_INFINITE; +static int keep_data_days = KEEP_INFINITE; + +/* restore configuration */ +static char *target_time; +static char *target_xid; +static char *target_inclusive; +static TimeLineID target_tli; + +/* show configuration */ +static bool show_all = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void parse_range(pgBackupRange *range, const char *arg1, const char *arg2); + +static pgut_option options[] = +{ + /* directory options */ + { 's', 'D', "pgdata" , &pgdata , SOURCE_ENV }, + { 's', 'A', "arclog-path" , &arclog_path , SOURCE_ENV }, + { 's', 'B', "backup-path" , &backup_path , SOURCE_ENV }, + { 's', 'S', "srvlog-path" , &srvlog_path , SOURCE_ENV }, + /* common options */ + { 'b', 'v', "verbose" , &verbose }, + { 'b', 'c', "check" , &check }, + /* backup options */ + { 'f', 'b', "backup-mode" , opt_backup_mode , SOURCE_ENV }, + { 'b', 's', "with-serverlog" , ¤t.with_serverlog , SOURCE_ENV }, + { 'b', 'Z', "compress-data" , ¤t.compress_data , SOURCE_ENV }, + { 'b', 'C', "smooth-checkpoint" , &smooth_checkpoint , SOURCE_ENV }, + /* options with only long name (keep-xxx) */ + { 'i', 1, "keep-data-generations" , &keep_data_generations, SOURCE_ENV }, + { 'i', 2, "keep-data-days" , &keep_data_days , SOURCE_ENV }, + { 'i', 3, "keep-arclog-files" , &keep_arclog_files , SOURCE_ENV }, + { 'i', 4, "keep-arclog-days" , &keep_arclog_days , SOURCE_ENV }, + { 'i', 5, "keep-srvlog-files" , &keep_srvlog_files , SOURCE_ENV }, + { 'i', 6, "keep-srvlog-days" , &keep_srvlog_days , SOURCE_ENV }, + /* restore options */ + { 's', 7, "recovery-target-time" , &target_time , SOURCE_ENV }, + { 's', 8, "recovery-target-xid" , &target_xid , SOURCE_ENV }, + { 's', 9, "recovery-target-inclusive" , &target_inclusive , SOURCE_ENV }, + { 'u', 10, "recovery-target-timeline" , &target_tli , SOURCE_ENV }, + /* catalog options */ + { 'b', 'a', "show-all" , &show_all }, + { 0 } +}; + +/* + * Entry point of pg_rman command. + */ +int +main(int argc, char *argv[]) +{ + const char *cmd = NULL; + const char *range1 = NULL; + const char *range2 = NULL; + bool show_timeline = false; + pgBackupRange range; + int i; + + /* do not buffer progress messages */ + setvbuf(stdout, 0, _IONBF, 0); /* TODO: remove this */ + + /* initialize configuration */ + catalog_init_config(¤t); + + /* overwrite configuration with command line arguments */ + i = pgut_getopt(argc, argv, options); + + for (; i < argc; i++) + { + if (cmd == NULL) + cmd = argv[i]; + else if (pg_strcasecmp(argv[i], "timeline") == 0 && + pg_strcasecmp(cmd, "show") == 0) + show_timeline = true; + else if (range1 == NULL) + range1 = argv[i]; + else if (range2 == NULL) + range2 = argv[i]; + else + elog(ERROR_ARGS, "too many arguments"); + } + + /* command argument (backup/restore/show/...) is required. */ + if (cmd == NULL) + { + help(false); + return HELP; + } + + /* get object range argument if any */ + if (range1 && range2) + parse_range(&range, range1, range2); + else if (range1) + parse_range(&range, range1, ""); + else + range.begin = range.end = 0; + + /* Read default configuration from file. */ + if (backup_path) + { + char path[MAXPGPATH]; + + snprintf(path, lengthof(path), "%s/%s", backup_path, PG_RMAN_INI_FILE); + pgut_readopt(path, options, ERROR_ARGS); + } + + /* BACKUP_PATH is always required */ + if (backup_path == NULL) + elog(ERROR_ARGS, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + + /* path must be absolute */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR_ARGS, "-D, --pgdata must be an absolute path"); + if (arclog_path != NULL && !is_absolute_path(arclog_path)) + elog(ERROR_ARGS, "-A, --arclog-path must be an absolute path"); + if (srvlog_path != NULL && !is_absolute_path(srvlog_path)) + elog(ERROR_ARGS, "-S, --srvlog-path must be an absolute path"); + + /* setup exclusion list for file search */ + for (i = 0; pgdata_exclude[i]; i++) /* find first empty slot */ + ; + if (arclog_path) + pgdata_exclude[i++] = arclog_path; + if (srvlog_path) + pgdata_exclude[i++] = srvlog_path; + + /* do actual operation */ + if (pg_strcasecmp(cmd, "init") == 0) + return do_init(); + else if (pg_strcasecmp(cmd, "backup") == 0) + return do_backup(smooth_checkpoint, + keep_arclog_files, keep_arclog_days, + keep_srvlog_files, keep_srvlog_days, + keep_data_generations, keep_data_days); + else if (pg_strcasecmp(cmd, "restore") == 0) + return do_restore(target_time, target_xid, target_inclusive, target_tli); + else if (pg_strcasecmp(cmd, "show") == 0) + return do_show(&range, show_timeline, show_all); + else if (pg_strcasecmp(cmd, "validate") == 0) + return do_validate(&range); + else if (pg_strcasecmp(cmd, "delete") == 0) + return do_delete(&range); + else + elog(ERROR_ARGS, "invalid command \"%s\"", cmd); + + return 0; +} + +void +pgut_help(bool details) +{ + printf(_("%s manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + printf(_("Usage:\n")); + printf(_(" %s OPTION init\n"), PROGRAM_NAME); + printf(_(" %s OPTION backup\n"), PROGRAM_NAME); + printf(_(" %s OPTION restore\n"), PROGRAM_NAME); + printf(_(" %s OPTION show [DATE]\n"), PROGRAM_NAME); + printf(_(" %s OPTION show timeline [DATE]\n"), PROGRAM_NAME); + printf(_(" %s OPTION validate [DATE]\n"), PROGRAM_NAME); + printf(_(" %s OPTION delete DATE\n"), PROGRAM_NAME); + + if (!details) + return; + + printf(_("\nCommon Options:\n")); + printf(_(" -D, --pgdata=PATH location of the database storage area\n")); + printf(_(" -A, --arclog-path=PATH location of archive WAL storage area\n")); + printf(_(" -S, --srvlog-path=PATH location of server log storage area\n")); + printf(_(" -B, --backup-path=PATH location of the backup storage area\n")); + printf(_(" -c, --check show what would have been done\n")); + printf(_("\nBackup options:\n")); + printf(_(" -b, --backup-mode=MODE full, incremental, or archive\n")); + printf(_(" -s, --with-serverlog also backup server log files\n")); + printf(_(" -Z, --compress-data compress data backup with zlib\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --keep-data-generations=N keep GENERATION of full data backup\n")); + printf(_(" --keep-data-days=DAY keep enough data backup to recover to DAY days age\n")); + printf(_(" --keep-arclog-files=NUM keep NUM of archived WAL\n")); + printf(_(" --keep-arclog-days=DAY keep archived WAL modified in DAY days\n")); + printf(_(" --keep-srvlog-files=NUM keep NUM of serverlogs\n")); + printf(_(" --keep-srvlog-days=DAY keep serverlog modified in DAY days\n")); + printf(_("\nRestore options:\n")); + printf(_(" --recovery-target-time time stamp up to which recovery will proceed\n")); + printf(_(" --recovery-target-xid transaction ID up to which recovery will proceed\n")); + printf(_(" --recovery-target-inclusive whether we stop just after the recovery target\n")); + printf(_(" --recovery-target-timeline recovering into a particular timeline\n")); + printf(_("\nCatalog options:\n")); + printf(_(" -a, --show-all show deleted backup too\n")); +} + +/* + * Create range object from one or two arguments. + * All not-digit characters in the argument(s) are igonred. + * Both arg1 and arg2 must be valid pointer. + */ +static void +parse_range(pgBackupRange *range, const char *arg1, const char *arg2) +{ + size_t len = strlen(arg1) + strlen(arg2) + 1; + char *tmp; + int num; + struct tm tm; + + tmp = pgut_malloc(len); + tmp[0] = '\0'; + if (arg1 != NULL) + remove_not_digit(tmp, len, arg1); + if (arg2 != NULL) + remove_not_digit(tmp + strlen(tmp), len - strlen(tmp), arg2); + + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + num = sscanf(tmp, "%04d %02d %02d %02d %02d %02d", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec); + + if (num < 1) + elog(ERROR_ARGS, _("supplied id(%s) is invalid."), tmp); + + free(tmp); + + /* adjust year and month to convert to time_t */ + tm.tm_year -= 1900; + if (num > 1) + tm.tm_mon -= 1; + range->begin = mktime(&tm); + + switch (num) + { + case 1: + tm.tm_year++; + break; + case 2: + tm.tm_mon++; + break; + case 3: + tm.tm_mday++; + break; + case 4: + tm.tm_hour++; + break; + case 5: + tm.tm_min++; + break; + case 6: + tm.tm_sec++; + break; + } + range->end = mktime(&tm); + range->end--; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg, ERROR_ARGS); +} diff --git a/pg_rman.h b/pg_rman.h new file mode 100644 index 00000000..20ca05df --- /dev/null +++ b/pg_rman.h @@ -0,0 +1,283 @@ +/*------------------------------------------------------------------------- + * + * pg_rman.h: Backup/Recovery manager for PostgreSQL. + * + * Copyright (c) 2009, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ +#ifndef PG_RMAN_H +#define PG_RMAN_H + +#include "postgres_fe.h" + +#include +#include "libpq-fe.h" + +#include "pgut/pgut.h" +#include "access/xlogdefs.h" +#include "utils/pg_crc.h" +#include "parray.h" + +/* Directory/File names */ +#define DATABASE_DIR "database" +#define ARCLOG_DIR "arclog" +#define SRVLOG_DIR "srvlog" +#define RESTORE_WORK_DIR "backup" +#define PG_XLOG_DIR "pg_xlog" +#define TIMELINE_HISTORY_DIR "timeline_history" +#define BACKUP_INI_FILE "backup.ini" +#define PG_RMAN_INI_FILE "pg_rman.ini" +#define MKDIRS_SH_FILE "mkdirs.sh" +#define DATABASE_FILE_LIST "file_database.txt" +#define ARCLOG_FILE_LIST "file_arclog.txt" +#define SRVLOG_FILE_LIST "file_srvlog.txt" + +/* Direcotry/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + +/* Exit code */ +#define ERROR_ARCHIVE_FAILED 20 /* cannot archive xlog file */ +#define ERROR_NO_BACKUP 21 /* backup was not found in the catalog */ +#define ERROR_CORRUPTED 22 /* backup catalog is corrupted */ +#define ERROR_ALREADY_RUNNING 23 /* another pg_rman is running */ +#define ERROR_PG_INCOMPATIBLE 24 /* block size is not compatible */ +#define ERROR_PG_RUNNING 25 /* PostgreSQL server is running */ +#define ERROR_PID_BROKEN 26 /* postmaster.pid file is broken */ + +/* backup mode file */ +typedef struct pgFile +{ + time_t mtime; /* time of last modification */ + mode_t mode; /* protection (file type and permission) */ + size_t size; /* size of the file */ + size_t read_size; /* size of the portion read (if only some pages are + backed up partially, it's different from size) */ + size_t write_size; /* size of the backed-up file. BYTES_INVALID means + that the file existed but was not backed up + because not modified since last backup. */ + pg_crc32 crc; /* CRC value of the file, regular file only */ + char *linked; /* path of the linked file */ + bool is_datafile; /* true if the file is PostgreSQL data file */ + char path[1]; /* path of the file */ +} pgFile; + +typedef struct pgBackupRange +{ + time_t begin; + time_t end; /* begin +1 when one backup is target */ +} pgBackupRange; + +#define pgBackupRangeIsValid(range) \ + (((range)->begin != (time_t) 0) || ((range)->end != (time_t) 0)) +#define pgBackupRangeIsSingle(range) \ + (pgBackupRangeIsValid(range) && (range)->begin == ((range)->end)) + +/* Backup status */ +/* XXX re-order ? */ +typedef enum BackupStatus +{ + BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ + BACKUP_STATUS_OK, /* completed backup */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_DELETING, /* data files are being deleted */ + BACKUP_STATUS_DELETED, /* data files have been deleted */ + BACKUP_STATUS_DONE, /* completed but not validated yet */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ +} BackupStatus; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID, + BACKUP_MODE_ARCHIVE, /* arhicve only */ + BACKUP_MODE_INCREMENTAL, /* incremental backup */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +/* + * pg_rman takes backup into the directroy $BACKUP_PATH//