From 2689ddb9ba9c5bf9af4157beadf7d4f05a5ef855 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 26 Feb 2018 17:55:39 +0300 Subject: [PATCH 01/99] use dynamically allocated array for backup_threads --- src/backup.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 08cc990d..93924dbc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -438,8 +438,9 @@ do_backup_instance(void) char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - pthread_t backup_threads[num_threads]; - backup_files_args *backup_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *backup_threads; + backup_files_args *backup_threads_args; pgBackup *prev_backup = NULL; char prev_backup_filelist_path[MAXPGPATH]; @@ -639,9 +640,12 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileCompareSize); /* init thread args with own file lists */ + backup_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + backup_threads_args = (backup_files_args *) palloc(sizeof(backup_files_args)*num_threads); + for (i = 0; i < num_threads; i++) { - backup_files_args *arg = pg_malloc(sizeof(backup_files_args)); + backup_files_args *arg = &(backup_threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; @@ -650,30 +654,29 @@ do_backup_instance(void) arg->prev_backup_start_lsn = prev_backup_start_lsn; arg->thread_backup_conn = NULL; arg->thread_cancel_conn = NULL; - backup_threads_args[i] = arg; } /* Run threads */ elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { + backup_files_args *arg = &(backup_threads_args[i]); elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, - backup_threads_args[i]); + arg); else pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) remote_backup_files, - backup_threads_args[i]); + arg); } /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); - pg_free(backup_threads_args[i]); } elog(LOG, "Data files are transfered"); From 1adde8f19f5e6abf2103a8e5a9bb1e9ab7caf8eb Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 26 Feb 2018 18:44:22 +0300 Subject: [PATCH 02/99] fix usage of symlink specific code --- src/backup.c | 4 ++++ src/dir.c | 16 +++++++++++++++- src/restore.c | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 93924dbc..6e636308 100644 --- a/src/backup.c +++ b/src/backup.c @@ -239,7 +239,11 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) else if (copybuf[156] == '2') { /* Symlink */ +#ifndef WIN32 pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif } else elog(ERROR, "Unrecognized link indicator \"%c\"\n", diff --git a/src/dir.c b/src/dir.c index 77b37274..eed62c5c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -380,7 +380,13 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, * Add to files list only files, links and directories. Skip sockets and * other unexpected file formats. */ - if (!S_ISDIR(file->mode) && !S_ISLNK(file->mode) && !S_ISREG(file->mode)) + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode) && +#ifndef WIN32 + !(S_ISLNK(file->mode)) +#else + !(pgwin32_is_junction(file->path)) +#endif + ) { elog(WARNING, "Skip file \"%s\": unexpected file format", file->path); return; @@ -430,7 +436,11 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, } /* chase symbolic link chain and find regular file or directory */ + #ifndef WIN32 while (S_ISLNK(file->mode)) + #else + while (pgwin32_is_junction(file->path)) + #endif { ssize_t len; char linked[MAXPGPATH]; @@ -718,7 +728,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); +#ifndef WIN32 if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif fprintf(out, ",\"linked\":\"%s\"", file->linked); fprintf(out, "}\n"); diff --git a/src/restore.c b/src/restore.c index 5965fe19..fb853b6a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -85,7 +85,7 @@ do_restore_or_validate(time_t target_backup_id, { int i; parray *backups; - parray *timelines; + parray *timelines = NULL; pgRecoveryTarget *rt = NULL; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; From 2a9baaa960db6782596be68011bdb80284548efe Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 12:31:41 +0300 Subject: [PATCH 03/99] Rename 'delete' in the ProbackupSubcmd enum to differ it from reserved keyword --- src/pg_probackup.c | 8 ++++---- src/pg_probackup.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 88270042..6fc55840 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -210,7 +210,7 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "show") == 0) backup_subcmd = SHOW; else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; + backup_subcmd = DELETE_SUBCMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG; else if (strcmp(argv[1], "show-config") == 0) @@ -256,7 +256,7 @@ main(int argc, char *argv[]) if (backup_subcmd == BACKUP || backup_subcmd == RESTORE || backup_subcmd == VALIDATE || - backup_subcmd == DELETE) + backup_subcmd == DELETE_SUBCMD) { int i, len = 0, @@ -377,7 +377,7 @@ main(int argc, char *argv[]) { if (backup_subcmd != RESTORE && backup_subcmd != VALIDATE - && backup_subcmd != DELETE + && backup_subcmd != DELETE_SUBCMD && backup_subcmd != SHOW) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -456,7 +456,7 @@ main(int argc, char *argv[]) false); case SHOW: return do_show(current.backup_id); - case DELETE: + case DELETE_SUBCMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3f34e8f9..1c1eef1b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -141,7 +141,7 @@ typedef enum ProbackupSubcmd RESTORE, VALIDATE, SHOW, - DELETE, + DELETE_SUBCMD, SET_CONFIG, SHOW_CONFIG } ProbackupSubcmd; From 18364caa34c71263fb575bc07012620274e66803 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 13:02:43 +0300 Subject: [PATCH 04/99] rewrite dir_create_dir() in a cross platform way --- src/dir.c | 8 +++++--- src/pg_probackup.h | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index eed62c5c..834b2708 100644 --- a/src/dir.c +++ b/src/dir.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include #include #include @@ -99,10 +98,12 @@ int dir_create_dir(const char *dir, mode_t mode) { char copy[MAXPGPATH]; - char parent[MAXPGPATH]; + char *parent; strncpy(copy, dir, MAXPGPATH); - strncpy(parent, dirname(copy), MAXPGPATH); + + parent = pstrdup(dir); + get_parent_directory(parent); /* Create parent first */ if (access(parent, F_OK) == -1) @@ -116,6 +117,7 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } + pfree(parent); return 0; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1c1eef1b..392ef579 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -28,6 +28,7 @@ #include "storage/checksum.h" #include "utils/pg_crc.h" #include "common/relpath.h" +#include "port.h" #include "utils/parray.h" #include "utils/pgut.h" From 7e096be2b3b440e92a63ef29d77a57ceeb4f64cc Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 14:37:46 +0300 Subject: [PATCH 05/99] add partial ptread implementation for windows port --- src/backup.c | 104 ++++++++++++++++++++++++++++++++++++++++++++- src/pg_probackup.h | 4 ++ src/restore.c | 6 ++- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6e636308..0ee2a9ba 100644 --- a/src/backup.c +++ b/src/backup.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "libpq/pqsignal.h" #include "storage/bufpage.h" @@ -29,6 +28,16 @@ #include "streamutil.h" #include "pgtar.h" +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; @@ -2763,3 +2772,96 @@ pg_ptrack_get_block(backup_files_args *arguments, return result; } + +#ifdef WIN32 /* WIN32 */ +volatile bool timer_exceeded = false; /* flag from signal handler */ + +static VOID CALLBACK +win32_timer_callback(PVOID lpParameter, BOOLEAN TimerOrWaitFired) +{ timer_exceeded = true; } + +static void +setalarm(int seconds) +{ + HANDLE queue; + HANDLE timer; + + /* This function will be called at most once, so we can cheat a bit. */ + queue = CreateTimerQueue(); + if (seconds > ((DWORD)-1) / 1000 || + !CreateTimerQueueTimer(&timer, queue, + win32_timer_callback, NULL, seconds * 1000, 0, + WT_EXECUTEINTIMERTHREAD | WT_EXECUTEONLYONCE)) + { + fprintf(stderr, "failed to set timer\n"); exit(1); + } +} + +/* partial pthread implementation for Windows */ + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} + +win32_pthread; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + th->result = th->routine(th->arg); + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, + win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 392ef579..7da50800 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -17,6 +17,9 @@ #ifndef WIN32 #include +#include +#else +#include "port/pthread-win32.h" #endif #include "access/timeline.h" @@ -30,6 +33,7 @@ #include "common/relpath.h" #include "port.h" + #include "utils/parray.h" #include "utils/pgut.h" diff --git a/src/restore.c b/src/restore.c index fb853b6a..c983ed0b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -14,10 +14,14 @@ #include #include #include -#include #include "catalog/pg_control.h" +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#endif + typedef struct { parray *files; From 834cfb6ab82c027dfd3ba8975d28b25fb5f7c066 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 14:40:05 +0300 Subject: [PATCH 06/99] fix 'unary minus operator applied to unsigned type' --- src/restore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index c983ed0b..a63a4bd3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -913,7 +913,8 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - entry->end = (uint32) (-1UL << 32) | -1UL; + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; parray_insert(result, 0, entry); return result; From 9ec02abc0df1843d81201a68c135407de4c55361 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 2 Mar 2018 15:22:57 +0300 Subject: [PATCH 07/99] WIP. Doesn't work. Cross platform multi threaded backup/restore --- Makefile | 7 +++++-- src/backup.c | 6 +++--- src/pg_probackup.h | 3 ++- src/restore.c | 4 ++-- src/validate.c | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 0f4fa672..e7106d55 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,10 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ - src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h \ + src/atomics.h -INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h src/atomics.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -61,6 +62,8 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ +src/atomics.h: $(top_srcdir)/src/include/port/atomics.h + rm -f $@ && $(LN_S) $(srchome)/src/include/port/atomics.h $@ ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c diff --git a/src/backup.c b/src/backup.c index 0ee2a9ba..77ca3311 100644 --- a/src/backup.c +++ b/src/backup.c @@ -383,7 +383,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); @@ -646,7 +646,7 @@ do_backup_instance(void) } /* setup threads */ - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* sort by size for load balancing */ @@ -1922,7 +1922,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7da50800..713b8e62 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -33,6 +33,7 @@ #include "common/relpath.h" #include "port.h" +#include "atomics.h" #include "utils/parray.h" #include "utils/pgut.h" @@ -105,7 +106,7 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile uint32 lock; /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; diff --git a/src/restore.c b/src/restore.c index a63a4bd3..24f6bfae 100644 --- a/src/restore.c +++ b/src/restore.c @@ -398,7 +398,7 @@ restore_backup(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ @@ -703,7 +703,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, diff --git a/src/validate.c b/src/validate.c index 2be75760..e6bb3b22 100644 --- a/src/validate.c +++ b/src/validate.c @@ -62,7 +62,7 @@ pgBackupValidate(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Validate files */ @@ -116,7 +116,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) From dff728e2720e5818bd8af4275378ac0fc998d81e Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Mar 2018 13:12:54 +0300 Subject: [PATCH 08/99] Add scripts for generate Visual Studio Project --- msvs/pg_probackup.sln | 28 ++++ msvs/template.pg_probackup.vcxproj | 203 ++++++++++++++++++++++ msvs/template.pg_probackup_2.vcxproj | 203 ++++++++++++++++++++++ win32build.pl | 240 +++++++++++++++++++++++++++ win32build_2.pl | 219 ++++++++++++++++++++++++ 5 files changed, 893 insertions(+) create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 win32build.pl create mode 100644 win32build_2.pl diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..3c616170 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..66c71b8b --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From 0c9bef39094eee3da418a73e09c3f827633d30a3 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 13:31:36 +0300 Subject: [PATCH 09/99] update VS project --- msvs/template.pg_probackup.vcxproj | 8 ++++---- msvs/template.pg_probackup_2.vcxproj | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 3c616170..8c731e28 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -68,22 +68,22 @@ true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj index 66c71b8b..2fc101a4 100644 --- a/msvs/template.pg_probackup_2.vcxproj +++ b/msvs/template.pg_probackup_2.vcxproj @@ -68,22 +68,22 @@ true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) From b7457d619fcad9f7696e409da1caa4f4f5f0816f Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 13:33:25 +0300 Subject: [PATCH 10/99] update VS project --- doit.cmd | 1 + 1 file changed, 1 insertion(+) create mode 100644 doit.cmd diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..cc29bc88 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file From b637a98b50e47ecee43a3e9715dc052f2a7df12d Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 14:53:04 +0300 Subject: [PATCH 11/99] include postgresql cross platform atomics. It takes ugly #undef FRONTEND hack to do that --- Makefile | 8 +++----- src/pg_probackup.h | 6 +++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e7106d55..f533d814 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,9 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ - src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h \ - src/atomics.h + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h -INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h src/atomics.h +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -62,8 +61,7 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -src/atomics.h: $(top_srcdir)/src/include/port/atomics.h - rm -f $@ && $(LN_S) $(srchome)/src/include/port/atomics.h $@ + ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 713b8e62..472b1d66 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -33,7 +33,11 @@ #include "common/relpath.h" #include "port.h" -#include "atomics.h" +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif #include "utils/parray.h" #include "utils/pgut.h" From f0547f46a158aa7eff415a2c495286a87468eea4 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 15:17:58 +0300 Subject: [PATCH 12/99] fix includes --- src/catalog.c | 1 - src/utils/logger.c | 8 +++++++- src/validate.c | 8 +++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 0e9b4de2..e82c434a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/src/utils/logger.c b/src/utils/logger.c index 5d51394f..3a9542d6 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -17,6 +16,13 @@ #include "logger.h" #include "pgut.h" +#ifndef WIN32 +#include +#include +#else +#include "port/pthread-win32.h" +#endif + /* Logger parameters */ int log_level_console = LOG_NONE; diff --git a/src/validate.c b/src/validate.c index e6bb3b22..e9f3bb9f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -11,9 +11,15 @@ #include "pg_probackup.h" #include -#include #include +#ifndef WIN32 +#include +#include +#else +#include "port/pthread-win32.h" +#endif + static void pgBackupValidateFiles(void *arg); static void do_validate_instance(void); From 3f91c372a5c377c89335b3994b28840b436540e1 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 15:18:15 +0300 Subject: [PATCH 13/99] update VS project --- msvs/template.pg_probackup.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 8c731e28..03caf86c 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -68,22 +68,22 @@ true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) From 990ccd32fc631178bb772d69f123e7dd861da870 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 15:24:49 +0300 Subject: [PATCH 14/99] use dynamically allocated array for restore_threads --- src/restore.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/restore.c b/src/restore.c index 24f6bfae..d8fd4221 100644 --- a/src/restore.c +++ b/src/restore.c @@ -349,8 +349,10 @@ restore_backup(pgBackup *backup) char list_path[MAXPGPATH]; parray *files; int i; - pthread_t restore_threads[num_threads]; - restore_files_args *restore_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *restore_threads; + restore_files_args *restore_threads_args; + if (backup->status != BACKUP_STATUS_OK) elog(ERROR, "Backup %s cannot be restored because it is not valid", @@ -394,6 +396,9 @@ restore_backup(pgBackup *backup) pgFileFree(parray_remove(files, i)); } + restore_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + restore_threads_args = (restore_files_args *) palloc(sizeof(restore_files_args)*num_threads); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -404,13 +409,13 @@ restore_backup(pgBackup *backup) /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = pg_malloc(sizeof(restore_files_args)); + restore_files_args *arg = &(restore_threads_args[i]); + arg->files = files; arg->backup = backup; elog(LOG, "Start thread for num:%li", parray_num(files)); - restore_threads_args[i] = arg; pthread_create(&restore_threads[i], NULL, (void *(*)(void *)) restore_files, arg); } @@ -418,9 +423,11 @@ restore_backup(pgBackup *backup) for (i = 0; i < num_threads; i++) { pthread_join(restore_threads[i], NULL); - pg_free(restore_threads_args[i]); } + pfree(restore_threads); + pfree(restore_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 14fa9542d43af91479ec8a75c86b8978ad8a6f03 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 17:02:58 +0300 Subject: [PATCH 15/99] use dynamically allocated array for validate_threads --- src/validate.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/validate.c b/src/validate.c index e9f3bb9f..96e06748 100644 --- a/src/validate.c +++ b/src/validate.c @@ -20,6 +20,16 @@ #include "port/pthread-win32.h" #endif +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + static void pgBackupValidateFiles(void *arg); static void do_validate_instance(void); @@ -41,8 +51,9 @@ pgBackupValidate(pgBackup *backup) char path[MAXPGPATH]; parray *files; bool corrupted = false; - pthread_t validate_threads[num_threads]; - validate_files_args *validate_threads_args[num_threads]; + /* arrays with meta info for multi threaded validate */ + pthread_t *validate_threads; + validate_files_args *validate_threads_args; int i; if (backup->status != BACKUP_STATUS_OK && @@ -71,25 +82,31 @@ pgBackupValidate(pgBackup *backup) pg_atomic_clear_flag(&file->lock); } + /* init thread args with own file lists */ + validate_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + validate_threads_args = (validate_files_args *) palloc(sizeof(validate_files_args)*num_threads); + /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = pg_malloc(sizeof(validate_files_args)); + validate_files_args *arg = &(validate_threads_args[i]); arg->files = files; arg->corrupted = false; - validate_threads_args[i] = arg; pthread_create(&validate_threads[i], NULL, (void *(*)(void *)) pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { + validate_files_args *arg = &(validate_threads_args[i]); pthread_join(validate_threads[i], NULL); - if (validate_threads_args[i]->corrupted) + if (arg->corrupted) corrupted = true; - pg_free(validate_threads_args[i]); } + pfree(validate_threads); + pfree(validate_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 351b3dd263e04c2e965ba9732c4abe63c0fba9ab Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 17:41:03 +0300 Subject: [PATCH 16/99] define pthread_t in logger.c for WINDOWS --- src/utils/logger.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/logger.c b/src/utils/logger.c index 3a9542d6..a380888e 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -23,6 +23,16 @@ #include "port/pthread-win32.h" #endif +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + /* Logger parameters */ int log_level_console = LOG_NONE; From dbd757761c1da90ef8bb52b114c5810b582396a8 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 18:45:52 +0300 Subject: [PATCH 17/99] update VS project --- msvs/template.pg_probackup.vcxproj | 14 ++++++++++---- src/data.c | 2 +- src/utils/logger.c | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 03caf86c..044a3e73 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -69,22 +69,26 @@ true ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + true ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + false ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + false ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + @@ -134,6 +138,7 @@ true true @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) libc;%(IgnoreSpecificDefaultLibraries) @@ -154,6 +159,7 @@ true true @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) libc;%(IgnoreSpecificDefaultLibraries) diff --git a/src/data.c b/src/data.c index 95966ace..6a76a844 100644 --- a/src/data.c +++ b/src/data.c @@ -853,7 +853,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out; + FILE *out=NULL; char buf[XLOG_BLCKSZ]; const char *to_path_p = to_path; char to_path_temp[MAXPGPATH]; diff --git a/src/utils/logger.c b/src/utils/logger.c index a380888e..5a069fa3 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -143,8 +143,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) bool write_to_file, write_to_error_log, write_to_stderr; - va_list error_args, - std_args; + va_list error_args=NULL, + std_args=NULL; time_t log_time = (time_t) time(NULL); char strfbuf[128]; From c7bfcdbcd359fe78f03a502f5fddcd47f0f20955 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Thu, 15 Mar 2018 17:49:08 +0300 Subject: [PATCH 18/99] changed char to unsigned char for sname in the struct pgut_option --- src/utils/pgut.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/pgut.h b/src/utils/pgut.h index def0f9d2..d956e138 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -58,7 +58,7 @@ typedef enum pgut_optsrc typedef struct pgut_option { char type; - char sname; /* short name */ + unsigned char sname; /* short name */ const char *lname; /* long name */ void *var; /* pointer to variable */ pgut_optsrc allowed; /* allowed source */ From e54b151df0bcf118e32da4552c5c5812864ad65c Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Sat, 17 Mar 2018 19:04:20 +0300 Subject: [PATCH 19/99] changes in the parsing command line options --- src/utils/pgut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6b5d9b6e..515f35c8 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) + if (opts[i].val > 128 || !isprint(opts[i].val)) continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1053,7 +1053,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + while ((c = getopt_long(argc>2 ? argc - 1 : argc, argc>2 ? argv + 1: argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); if (opt && opt->allowed < SOURCE_CMDLINE) From 84358f6963de6bb8f8766ac85ba7e9c9c1877c53 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Sat, 17 Mar 2018 19:22:26 +0300 Subject: [PATCH 20/99] changes in the parsing command line options --- src/utils/pgut.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 515f35c8..1573da0f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1046,6 +1046,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; + len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1053,7 +1054,8 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - while ((c = getopt_long(argc>2 ? argc - 1 : argc, argc>2 ? argv + 1: argv, optstring, longopts, &optindex)) != -1) + optind = 2; + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); if (opt && opt->allowed < SOURCE_CMDLINE) From d0092bb3c951816087a73f1b396c3c12789495fa Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 20 Mar 2018 11:15:53 +0300 Subject: [PATCH 21/99] changes in the parsing command line options --- src/utils/pgut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 1573da0f..08c54b11 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (opts[i].val > 128 || !isprint(opts[i].val)) + if (!isprint(opts[i].val)) //opts[i].val > 128 || continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) From 7e8cffb23fb86bff2cb336e66796bcade1147faa Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Thu, 22 Mar 2018 18:22:23 +0300 Subject: [PATCH 22/99] some changes --- src/backup.c | 17 +++++++++++++---- src/configure.c | 3 ++- src/dir.c | 4 +++- src/pg_probackup.c | 1 + src/pg_probackup.h | 10 ++++++++++ src/utils/pgut.c | 2 +- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 77ca3311..5c792933 100644 --- a/src/backup.c +++ b/src/backup.c @@ -552,9 +552,16 @@ do_backup_instance(void) join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); dir_create_dir(dst_backup_path, DIR_PERMISSION); + if (start_stream_mut == NULL){ + pthread_mutex_init(&start_stream_mut,NULL); + } + pthread_mutex_lock(&start_stream_mut); pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); pthread_mutex_lock(&start_stream_mut); +#ifdef WIN32 + sleep(10); +#endif if (conn == NULL) elog(ERROR, "Cannot continue backup because stream connect has failed."); @@ -936,7 +943,7 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - + if (system_id_conn != system_identifier) elog(ERROR, "Backup data directory was initialized for system id %ld, but connected instance system id is %ld", system_identifier, system_id_conn); @@ -959,14 +966,15 @@ confirm_block_size(const char *name, int blcksz) res = pgut_execute(backup_conn, "SELECT current_setting($1)", 1, &name, true); if (PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); - + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - PQclear(res); - if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); + + PQclear(res);//bad pointer to endp + } /* @@ -2864,4 +2872,5 @@ pthread_join(pthread_t th, void **thread_return) return 0; } + #endif /* WIN32 */ diff --git a/src/configure.c b/src/configure.c index 55d2bbc5..fa165190 100644 --- a/src/configure.c +++ b/src/configure.c @@ -114,7 +114,8 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); + //fprintf(out, "system-identifier = %li\n", config->system_identifier); + fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/dir.c b/src/dir.c index 834b2708..2069964a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -150,6 +150,8 @@ pgFileInit(const char *path) pgFile *file; file = (pgFile *) pgut_malloc(sizeof(pgFile)); + file->name = 0; + file->size = 0; file->mode = 0; file->read_size = 0; @@ -520,7 +522,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, break; } } - else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + else if (file->name!=0 && strcmp(file->name, pgdata_exclude_dir[i]) == 0) { skip = true; break; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6fc55840..0206da22 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -434,6 +434,7 @@ main(int argc, char *argv[]) start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 472b1d66..cb84d4b4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -506,5 +506,15 @@ extern void pgBackup_init(pgBackup *backup); /* in status.c */ extern bool is_pg_running(void); +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#endif +#endif #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 08c54b11..de006657 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1221,7 +1221,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "# \n\r\t\v"); + i = j = strcspn(s, "#\n\r\t\v");//removed space memcpy(dst, s, j); } From e7665c1d504edca2b7784650af5490957fb2ca3d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 10 Apr 2018 18:44:51 +0300 Subject: [PATCH 23/99] Arthur Zakirov: fopen() in binary mode --- src/data.c | 8 ++++---- src/pg_probackup.h | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index 6a76a844..6934e627 100644 --- a/src/data.c +++ b/src/data.c @@ -865,7 +865,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, "rb"); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -896,7 +896,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, "wb"); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1018,7 +1018,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, "rb"); if (in == NULL) { #ifdef HAVE_LIBZ @@ -1055,7 +1055,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, "wb"); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index cb84d4b4..9a3d8e8c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -514,6 +514,7 @@ extern bool is_pg_running(void); #define read _read #define close _close #define write _write +#define mkdir(dir,mode) _mkdir(dir) #endif #endif From 03412d766f6b754413412668dfcf5d2694874413 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 24 Apr 2018 12:59:25 +0300 Subject: [PATCH 24/99] Add build script and project template for the 9.6 version --- doit96.cmd | 1 + msvs/template.pg_probackup96.vcxproj | 207 +++++++++++++++++++++++ win32build96.pl | 240 +++++++++++++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 doit96.cmd create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 win32build96.pl diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..c095d7cc --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,207 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + From 9a1e5ba0445384c9e531b208a42d304b2ef861b4 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 26 Apr 2018 15:02:03 +0300 Subject: [PATCH 25/99] Function pg_atomic_test_set_flag returns true if the flag has successfully been set, false otherwise --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5c792933..05a5c6df 100644 --- a/src/backup.c +++ b/src/backup.c @@ -692,7 +692,7 @@ do_backup_instance(void) (void *(*)(void *)) remote_backup_files, arg); } - + /* Wait threads */ for (i = 0; i < num_threads; i++) { @@ -1930,7 +1930,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ From 83d64c5b9d0d2ee4c7cc5e9a128108799bb279de Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 26 Apr 2018 15:23:50 +0300 Subject: [PATCH 26/99] Function pg_atomic_test_set_flag returns true if the flag has successfully been set, false otherwise --- src/backup.c | 2 +- src/restore.c | 2 +- src/validate.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 05a5c6df..0c65f728 100644 --- a/src/backup.c +++ b/src/backup.c @@ -383,7 +383,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); diff --git a/src/restore.c b/src/restore.c index d8fd4221..d91678cb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -710,7 +710,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, diff --git a/src/validate.c b/src/validate.c index 96e06748..39c40bdd 100644 --- a/src/validate.c +++ b/src/validate.c @@ -139,7 +139,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) From ac98fefc6e6b0d45df28efe2f6eefa1c39bf4510 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Fri, 11 May 2018 18:17:54 +0300 Subject: [PATCH 27/99] change mode for fopen w->PG_BINARY_W, r->PG_BINARY_R --- src/backup.c | 7 ++++--- src/data.c | 14 +++++++------- src/dir.c | 4 ++-- src/pg_probackup.c | 2 +- src/status.c | 2 +- src/utils/logger.c | 4 ++-- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0c65f728..f39a79f3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -287,7 +287,7 @@ static void remote_copy_file(PGconn *conn, pgFile* file) DATABASE_DIR); join_path_components(to_path, database_path, file->path); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -562,6 +562,7 @@ do_backup_instance(void) #ifdef WIN32 sleep(10); #endif + if (conn == NULL) elog(ERROR, "Cannot continue backup because stream connect has failed."); @@ -1715,7 +1716,7 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, "w"); + fp = fopen(backup_label, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1763,7 +1764,7 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, "w"); + fp = fopen(tablespace_map, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); diff --git a/src/data.c b/src/data.c index 6934e627..0d4ba525 100644 --- a/src/data.c +++ b/src/data.c @@ -421,7 +421,7 @@ backup_data_file(backup_files_args* arguments, INIT_CRC32C(file->crc); /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(file->crc); @@ -455,7 +455,7 @@ backup_data_file(backup_files_args* arguments, /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -547,7 +547,7 @@ restore_data_file(const char *from_root, BlockNumber blknum; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { elog(ERROR, "cannot open backup file \"%s\": %s", file->path, @@ -562,7 +562,7 @@ restore_data_file(const char *from_root, join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, "r+"); if (out == NULL && errno == ENOENT) - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -694,7 +694,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); @@ -710,7 +710,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -1189,7 +1189,7 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); diff --git a/src/dir.c b/src/dir.c index 2069964a..cf825542 100644 --- a/src/dir.c +++ b/src/dir.c @@ -216,7 +216,7 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, "r"); + fp = fopen(file->path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", file->path, strerror(errno)); @@ -333,7 +333,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, "r"); + black_list_file = fopen(path, PG_BINARY_R); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0206da22..de4394a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -181,13 +181,13 @@ main(int argc, char *argv[]) /* Check if backup_path is directory. */ struct stat stat_buf; int rc; - /* initialize configuration */ pgBackup_init(¤t); PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); + /* Parse subcommands and non-subcommand options */ if (argc > 1) { diff --git a/src/status.c b/src/status.c index 1c7c6038..155a07f4 100644 --- a/src/status.c +++ b/src/status.c @@ -38,7 +38,7 @@ get_pgpid(void) snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); - pidf = fopen(pid_file, "r"); + pidf = fopen(pid_file, PG_BINARY_R); if (pidf == NULL) { /* No pid file, not an error on startup */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 5a069fa3..09cad75d 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -515,7 +515,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, "r"); + control_file = fopen(control, PG_BINARY_R); if (control_file == NULL) elog(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -561,7 +561,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, "w"); + control_file = fopen(control, PG_BINARY_W); if (control_file == NULL) elog(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); From ceaa2ce9cdf64b13f52df42f23a5640d339aeda5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 21 May 2018 19:06:12 +0300 Subject: [PATCH 28/99] PGPRO-533: Allow to show backup list in json format --- src/backup.c | 24 ++- src/catalog.c | 64 +++++- src/help.c | 2 + src/pg_probackup.c | 63 +++--- src/pg_probackup.h | 20 +- src/show.c | 494 +++++++++++++++++++++++++++++++++++++-------- src/util.c | 13 +- src/utils/pgut.h | 2 +- 8 files changed, 532 insertions(+), 150 deletions(-) diff --git a/src/backup.c b/src/backup.c index 57c4be51..6a2333bd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -633,7 +633,7 @@ do_backup_instance(void) * For backup from master wait for previous segment. * For backup from replica wait for current segment. */ - !from_replica, backup_files_list); + !current.from_replica, backup_files_list); } if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) @@ -815,11 +815,15 @@ do_backup(time_t start_time) pgut_atexit_push(backup_disconnect, NULL); current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); confirm_block_size("wal_block_size", XLOG_BLCKSZ); - from_replica = pg_is_in_recovery(); + current.from_replica = pg_is_in_recovery(); /* Confirm that this server version is supported */ check_server_version(); @@ -859,7 +863,7 @@ do_backup(time_t start_time) } } - if (from_replica) + if (current.from_replica) { /* Check master connection options */ if (master_host == NULL) @@ -956,7 +960,7 @@ check_server_version(void) "server version is %s, must be %s or higher", server_version_str, "9.5"); - if (from_replica && server_version < 90600) + if (current.from_replica && server_version < 90600) elog(ERROR, "server version is %s, must be %s or higher for backup from replica", server_version_str, "9.6"); @@ -1061,7 +1065,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; /* For replica we call pg_start_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (backup->from_replica) ? master_conn : backup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1106,7 +1110,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) } /* Wait for start_lsn to be replayed by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(backup->start_lsn, true); /* @@ -1554,8 +1558,6 @@ wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) { uint32 try_count = 0; - Assert(from_replica); - while (true) { PGresult *res; @@ -1650,7 +1652,7 @@ pg_stop_backup(pgBackup *backup) elog(FATAL, "backup is not in progress"); /* For replica we call pg_stop_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (current.from_replica) ? master_conn : backup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", @@ -1663,7 +1665,7 @@ pg_stop_backup(pgBackup *backup) const char *params[1]; char name[1024]; - if (!from_replica) + if (!current.from_replica) snprintf(name, lengthof(name), "pg_probackup, backup_id %s", base36enc(backup->start_time)); else @@ -1891,7 +1893,7 @@ pg_stop_backup(pgBackup *backup) stream_xlog_path[MAXPGPATH]; /* Wait for stop_lsn to be received by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. diff --git a/src/catalog.c b/src/catalog.c index f5884f01..6c9d36b5 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -385,10 +385,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "#Configuration\n"); fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); - fprintf(out, "stream = %s\n", backup->stream?"true":"false"); - fprintf(out, "compress-alg = %s\n", deparse_compress_alg(compress_alg)); - fprintf(out, "compress-level = %d\n", compress_level); - fprintf(out, "from-replica = %s\n", from_replica?"true":"false"); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); @@ -429,7 +430,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->data_bytes != BYTES_INVALID) fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); - if (backup->data_bytes != BYTES_INVALID) + if (backup->wal_bytes != BYTES_INVALID) fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); fprintf(out, "status = %s\n", status2str(backup->status)); @@ -475,10 +476,8 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; - char *compress_alg = NULL; char *server_version = NULL; - int *compress_level; - bool *from_replica; + char *compress_alg = NULL; pgut_option options[] = { @@ -500,8 +499,8 @@ readBackupControlFile(const char *path) {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, - {'u', 0, "compress-level", &compress_level, SOURCE_FILE_STRICT}, - {'b', 0, "from-replica", &from_replica, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {0} }; @@ -578,6 +577,9 @@ readBackupControlFile(const char *path) pfree(server_version); } + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + return backup; } @@ -626,6 +628,48 @@ deparse_backup_mode(BackupMode mode) return NULL; } +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + /* free pgBackup object */ void pgBackupFree(void *backup) diff --git a/src/help.c b/src/help.c index 2f84b225..a41e8167 100644 --- a/src/help.c +++ b/src/help.c @@ -128,6 +128,7 @@ help_pg_probackup(void) printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired]\n")); @@ -362,6 +363,7 @@ help_show(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5d464171..efb8e6a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -47,7 +47,6 @@ char *replication_slot = NULL; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -bool from_replica = false; bool is_remote_backup = false; /* Wait timeout for WAL segment archiving */ uint32 archive_timeout = 300; /* default is 300 seconds */ @@ -83,7 +82,7 @@ uint32 retention_window = 0; /* compression options */ CompressAlg compress_alg = NOT_DEFINED_COMPRESS; int compress_level = DEFAULT_COMPRESS_LEVEL; -bool compress_shortcut = false; +bool compress_shortcut = false; /* other options */ char *instance_name; @@ -94,6 +93,9 @@ static char *wal_file_path; static char *wal_file_name; static bool file_overwrite = false; +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + /* current settings */ pgBackup current; ProbackupSubcmd backup_subcmd; @@ -104,6 +106,7 @@ static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); static void compress_init(void); @@ -178,6 +181,8 @@ static pgut_option options[] = { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, { 0 } }; @@ -517,49 +522,31 @@ opt_log_level_file(pgut_option *opt, const char *arg) log_level_file = parse_log_level(arg); } -CompressAlg -parse_compress_alg(const char *arg) +static void +opt_show_format(pgut_option *opt, const char *arg) { + const char *v = arg; size_t len; /* Skip all spaces detected */ - while (isspace((unsigned char)*arg)) - arg++; - len = strlen(arg); + while (IsSpace(*v)) + v++; + len = strlen(v); - if (len == 0) - elog(ERROR, "compress algrorithm is empty"); - - if (pg_strncasecmp("zlib", arg, len) == 0) - return ZLIB_COMPRESS; - else if (pg_strncasecmp("pglz", arg, len) == 0) - return PGLZ_COMPRESS; - else if (pg_strncasecmp("none", arg, len) == 0) - return NONE_COMPRESS; - else - elog(ERROR, "invalid compress algorithm value \"%s\"", arg); - - return NOT_DEFINED_COMPRESS; -} - -const char* -deparse_compress_alg(int alg) -{ - switch (alg) + if (len > 0) { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - return "none"; - case ZLIB_COMPRESS: - return "zlib"; - case PGLZ_COMPRESS: - return "pglz"; + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } - - return NULL; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } -void +static void opt_compress_alg(pgut_option *opt, const char *arg) { compress_alg = parse_compress_alg(arg); @@ -568,8 +555,8 @@ opt_compress_alg(pgut_option *opt, const char *arg) /* * Initialize compress and sanity checks for compress. */ -static -void compress_init(void) +static void +compress_init(void) { /* Default algorithm is zlib */ if (compress_shortcut) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ce..0fefef1b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -149,6 +149,12 @@ typedef enum ProbackupSubcmd SHOW_CONFIG } ProbackupSubcmd; +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ @@ -214,6 +220,9 @@ typedef struct pgBackup /* Size of WAL files in archive needed to restore this backup */ int64 wal_bytes; + CompressAlg compress_alg; + int compress_level; + /* Fields needed for compatibility check */ uint32 block_size; uint32 wal_block_size; @@ -221,13 +230,14 @@ typedef struct pgBackup char server_version[100]; - bool stream; /* Was this backup taken in stream mode? + bool stream; /* Was this backup taken in stream mode? * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ time_t parent_backup; /* Identifier of the previous backup. * Which is basic backup for this * incremental backup. */ - char *primary_conninfo; /* Connection parameters of the backup - * in the format suitable for recovery.conf */ + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ } pgBackup; /* Recovery target for restore and validate subcommands */ @@ -310,7 +320,6 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; extern uint32 archive_timeout; -extern bool from_replica; extern bool is_remote_backup; extern const char *master_db; extern const char *master_host; @@ -348,6 +357,9 @@ extern const char* deparse_compress_alg(int alg); extern char *instance_name; extern uint64 system_identifier; +/* show options */ +extern ShowFormat show_format; + /* current settings */ extern pgBackup current; extern ProbackupSubcmd backup_subcmd; diff --git a/src/show.c b/src/show.c index b6eee867..b97d9427 100644 --- a/src/show.c +++ b/src/show.c @@ -3,28 +3,38 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" + #include #include #include #include +#include "pqexpbuffer.h" -static void show_backup_list(FILE *out, parray *backup_list); -static void show_backup_detail(FILE *out, pgBackup *backup); -static int do_show_instance(time_t requested_backup_id); + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static uint8 json_level = 0; int do_show(time_t requested_backup_id) { - - if (instance_name == NULL - && requested_backup_id != INVALID_BACKUP_ID) + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use --backup_id option"); if (instance_name == NULL) @@ -38,10 +48,12 @@ do_show(time_t requested_backup_id) join_path_components(path, backup_path, BACKUPS_DIR); dir = opendir(path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); - errno = 0; - while ((dent = readdir(dir))) + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) { char child[MAXPGPATH]; struct stat st; @@ -54,73 +66,47 @@ do_show(time_t requested_backup_id) join_path_components(child, path, dent->d_name); if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; instance_name = dent->d_name; sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - fprintf(stdout, "\nBACKUP INSTANCE '%s'\n", instance_name); - do_show_instance(0); + + show_instance(INVALID_BACKUP_ID, true); } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + return 0; } else - return do_show_instance(requested_backup_id); -} - -/* - * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information - * about all backups in the backup instance. - * If valid backup id is passed, show detailed meta information - * about specified backup. - */ -static int -do_show_instance(time_t requested_backup_id) -{ - if (requested_backup_id != INVALID_BACKUP_ID) - { - pgBackup *backup; - - backup = read_backup(requested_backup_id); - if (backup == NULL) - { - elog(INFO, "Requested backup \"%s\" is not found.", - /* We do not need free base36enc's result, we exit anyway */ - base36enc(requested_backup_id)); - /* This is not error */ - return 0; - } - - show_backup_detail(stdout, backup); - - /* cleanup */ - pgBackupFree(backup); - - } - else - { - parray *backup_list; - - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); - - show_backup_list(stdout, backup_list); - - /* cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } - - return 0; + return show_backup(requested_backup_id); } static void pretty_size(int64 size, char *buf, size_t len) { - int exp = 0; + int exp = 0; /* minus means the size is invalid */ if (size < 0) @@ -219,16 +205,113 @@ get_parent_tli(TimeLineID child_tli) return result; } +/* + * Initialize instance visualization. + */ static void -show_backup_list(FILE *out, parray *backup_list) +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + if (backup_list == NULL) + elog(ERROR, "Failed to get backup list."); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) { int i; + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + /* if you add new fields here, fix the header */ /* show header */ - fputs("============================================================================================================================================\n", out); - fputs(" Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); - fputs("============================================================================================================================================\n", out); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); for (i = 0; i < parray_num(backup_list); i++) { @@ -255,27 +338,270 @@ show_backup_list(FILE *out, parray *backup_list) /* Get parent timeline before printing */ parent_tli = get_parent_tli(backup->tli); - fprintf(out, " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", - instance_name, - (backup->server_version[0] ? backup->server_version : "----"), - base36enc(backup->start_time), - timestamp, - pgBackupGetBackupMode(backup), - backup->stream ? "STREAM": "ARCHIVE", - backup->tli, - parent_tli, - duration, - data_bytes_str, - (uint32) (backup->start_lsn >> 32), - (uint32) backup->start_lsn, - (uint32) (backup->stop_lsn >> 32), - (uint32) backup->stop_lsn, - status2str(backup->status)); + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +static void +json_add_indent(PQExpBuffer buf) +{ + uint8 i; + + if (json_level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < json_level; i++) + appendPQExpBufferStr(buf, " "); +} + +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +static void +json_add(PQExpBuffer buf, JsonToken type) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + json_level++; + break; + case JT_END_ARRAY: + json_level--; + if (json_level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf); + appendPQExpBufferChar(buf, '{'); + json_level++; + break; + case JT_END_OBJECT: + json_level--; + if (json_level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; } } static void -show_backup_detail(FILE *out, pgBackup *backup) +json_add_escaped(PQExpBuffer buf, const char *str) { - pgBackupWriteControl(out, backup); + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} + +static void +json_add_key(PQExpBuffer buf, const char *name, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +static void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + bool add_comma) +{ + json_add_key(buf, name, add_comma); + json_add_escaped(buf, value); +} + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT); + + json_add_value(buf, "instance-name", instance_name, false); + + json_add_key(buf, "backups", true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT); + + json_add_value(buf, "id", base36enc(backup->start_time), true); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), true); + + json_add_key(buf, "compress-level", true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", true); + + json_add_key(buf, "block-size", true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "server-version", backup->server_version, true); + + json_add_key(buf, "current-tli", true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, true); + + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, true); + + json_add_key(buf, "recovery-xid", true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, true); + + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + json_add_value(buf, "data-bytes", data_bytes_str, true); + + pretty_size(backup->wal_bytes, data_bytes_str, + lengthof(data_bytes_str)); + json_add_value(buf, "wal-bytes", data_bytes_str, true); + + if (backup->end_time != (time_t) 0) + { + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + json_add_value(buf, "time", duration, true); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, true); + + json_add_value(buf, "status", status2str(backup->status), true); + + json_add(buf, JT_END_OBJECT); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT); + + first_instance = false; } diff --git a/src/util.c b/src/util.c index f2c84f6e..7aeba211 100644 --- a/src/util.c +++ b/src/util.c @@ -176,8 +176,8 @@ uint32 get_data_checksum_version(bool safe) { ControlFileData ControlFile; - char *buffer; - size_t size; + char *buffer; + size_t size; /* First fetch file... */ buffer = slurpFile(pgdata, "global/pg_control", &size, safe); @@ -310,10 +310,19 @@ pgBackup_init(pgBackup *backup) backup->end_time = (time_t) 0; backup->recovery_xid = 0; backup->recovery_time = (time_t) 0; + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = NOT_DEFINED_COMPRESS; + backup->compress_level = 0; + backup->block_size = BLCKSZ; backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + backup->stream = false; + backup->from_replica = false; backup->parent_backup = 0; backup->primary_conninfo = NULL; backup->server_version[0] = '\0'; diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a9003f2f..803d2c57 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -59,7 +59,7 @@ typedef enum pgut_optsrc typedef struct pgut_option { char type; - char sname; /* short name */ + uint8 sname; /* short name */ const char *lname; /* long name */ void *var; /* pointer to variable */ pgut_optsrc allowed; /* allowed source */ From 6832cbbcb90c43a378e70771079dce2fad94b2d3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 22 May 2018 15:55:18 +0300 Subject: [PATCH 29/99] Replaced the backslash in the path to the backup --- src/restore.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/restore.c b/src/restore.c index d91678cb..71efaca5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -787,6 +787,14 @@ create_recovery_conf(time_t backup_id, fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); +#ifdef WIN32 + char *p1 = backup_path; + while (*p1){ + if ((*p1) == '\\') + *p1 = '/'; + p1++; + } +#endif fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s --wal-file-path %%p --wal-file-name %%f'\n", PROGRAM_NAME, backup_path, instance_name); fprintf(fp, "recovery_target_action = 'promote'\n"); From e71312cc03adb35e6e0c829ee87248db9b069f78 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 30 May 2018 20:56:01 +0300 Subject: [PATCH 30/99] add parent_backup_link to pgBackup structure. No usage for now --- src/catalog.c | 27 +++++++++++++++++++++++++++ src/pg_probackup.h | 3 +++ src/restore.c | 1 + src/util.c | 1 + src/validate.c | 1 + 5 files changed, 33 insertions(+) diff --git a/src/catalog.c b/src/catalog.c index f5884f01..45e080c2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -256,6 +256,8 @@ catalog_get_backup_list(time_t requested_backup_id) parray *backups = NULL; pgBackup *backup = NULL; + int i; + /* open backup instance backups directory */ date_dir = opendir(backup_instance_path); if (date_dir == NULL) @@ -283,6 +285,7 @@ catalog_get_backup_list(time_t requested_backup_id) /* read backup information from BACKUP_CONTROL_FILE */ snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); + backup->backup_id = backup->start_time; /* ignore corrupted backups */ if (backup) @@ -316,6 +319,30 @@ catalog_get_backup_list(time_t requested_backup_id) parray_qsort(backups, pgBackupCompareIdDesc); + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + return backups; err_proc: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ce..f6eae19f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -184,6 +184,8 @@ typedef struct pgBackupConfig int compress_level; } pgBackupConfig; +typedef struct pgBackup pgBackup; + /* Information about single backup stored in backup.conf */ typedef struct pgBackup { @@ -226,6 +228,7 @@ typedef struct pgBackup time_t parent_backup; /* Identifier of the previous backup. * Which is basic backup for this * incremental backup. */ + pgBackup *parent_backup_link; char *primary_conninfo; /* Connection parameters of the backup * in the format suitable for recovery.conf */ } pgBackup; diff --git a/src/restore.c b/src/restore.c index 69bac841..52378168 100644 --- a/src/restore.c +++ b/src/restore.c @@ -203,6 +203,7 @@ do_restore_or_validate(time_t target_backup_id, } /* If we already found dest_backup, look for full backup. */ + /* TODO Now, as we have all backups linked, we can probably get rid of that?"*/ if (dest_backup) { if (current_backup->backup_mode == BACKUP_MODE_FULL) diff --git a/src/util.c b/src/util.c index f2c84f6e..ba5647d9 100644 --- a/src/util.c +++ b/src/util.c @@ -315,6 +315,7 @@ pgBackup_init(pgBackup *backup) backup->wal_block_size = XLOG_BLCKSZ; backup->stream = false; backup->parent_backup = 0; + backup->parent_backup_link = NULL; backup->primary_conninfo = NULL; backup->server_version[0] = '\0'; } diff --git a/src/validate.c b/src/validate.c index afc7f07d..b4a1c839 100644 --- a/src/validate.c +++ b/src/validate.c @@ -283,6 +283,7 @@ do_validate_instance(void) elog(ERROR, "Failed to get backup list."); /* Valiate each backup along with its xlog files. */ + /* TODO Maybe use parent_backup_link instead of looking for backups in the list */ for (i = 0; i < parray_num(backups); i++) { pgBackup *base_full_backup = NULL; From 6df8c2aaec213d0e149644df53606ab307cda336 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 31 May 2018 20:31:12 +0300 Subject: [PATCH 31/99] PGPRO-533: Replace instance-name by instance, fix typos --- src/show.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/show.c b/src/show.c index b97d9427..98dab419 100644 --- a/src/show.c +++ b/src/show.c @@ -494,7 +494,7 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); - json_add_value(buf, "instance-name", instance_name, false); + json_add_value(buf, "instance", instance_name, false); json_add_key(buf, "backups", true); @@ -517,13 +517,13 @@ show_instance_json(parray *backup_list) json_add(buf, JT_BEGIN_OBJECT); - json_add_value(buf, "id", base36enc(backup->start_time), true); + json_add_value(buf, "id", base36enc(backup->start_time), false); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", base36enc(backup->parent_backup), true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); From 733354c40f0e7540d781ea9240b728494d8b3119 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 1 Jun 2018 17:09:32 +0300 Subject: [PATCH 32/99] PGPRO-533: Reformat json output, add program-version to backup.control --- src/catalog.c | 10 ++++++++ src/pg_probackup.h | 1 + src/show.c | 58 ++++++++++++++++++++++++++++++++-------------- src/util.c | 1 + 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 6c9d36b5..eb5d3be7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -395,6 +395,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "block-size = %u\n", backup->block_size); fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); if (backup->server_version[0] != '\0') fprintf(out, "server-version = %s\n", backup->server_version); @@ -476,6 +477,7 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; + char *program_version = NULL; char *server_version = NULL; char *compress_alg = NULL; @@ -494,6 +496,7 @@ readBackupControlFile(const char *path) {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, @@ -570,6 +573,13 @@ readBackupControlFile(const char *path) free(parent_backup); } + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + if (server_version) { StrNCpy(backup->server_version, server_version, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0fefef1b..d3ce8241 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -228,6 +228,7 @@ typedef struct pgBackup uint32 wal_block_size; uint32 checksum_version; + char program_version[100]; char server_version[100]; bool stream; /* Was this backup taken in stream mode? diff --git a/src/show.c b/src/show.c index 98dab419..03ae6488 100644 --- a/src/show.c +++ b/src/show.c @@ -26,6 +26,21 @@ static int show_backup(time_t requested_backup_id); static void show_instance_plain(parray *backup_list, bool show_name); static void show_instance_json(parray *backup_list); +/* Json output functions */ + +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +static void json_add(PQExpBuffer buf, JsonToken type); +static void json_add_key(PQExpBuffer buf, const char *name, bool add_comma); +static void json_add_value(PQExpBuffer buf, const char *name, const char *value, + bool add_comma); + static PQExpBufferData show_buf; static bool first_instance = true; static uint8 json_level = 0; @@ -219,8 +234,9 @@ show_instance_start(void) first_instance = true; json_level = 0; - appendPQExpBufferChar(&show_buf, '['); - json_level++; + json_add(&show_buf, JT_BEGIN_OBJECT); + json_add_key(&show_buf, "instances", false); + json_add(&show_buf, JT_BEGIN_ARRAY); } /* @@ -230,7 +246,11 @@ static void show_instance_end(void) { if (show_format == SHOW_JSON) - appendPQExpBufferStr(&show_buf, "\n]\n"); + { + json_add(&show_buf, JT_END_ARRAY); + json_add(&show_buf, JT_END_OBJECT); + appendPQExpBufferChar(&show_buf, '\n'); + } fputs(show_buf.data, stdout); termPQExpBuffer(&show_buf); @@ -375,14 +395,6 @@ json_add_indent(PQExpBuffer buf) appendPQExpBufferStr(buf, " "); } -typedef enum -{ - JT_BEGIN_ARRAY, - JT_END_ARRAY, - JT_BEGIN_OBJECT, - JT_END_OBJECT -} JsonToken; - static void json_add(PQExpBuffer buf, JsonToken type) { @@ -493,10 +505,10 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, instance_name, false); - json_add_value(buf, "instance", instance_name, false); - - json_add_key(buf, "backups", true); + json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, "backups", false); /* * List backups. @@ -516,14 +528,19 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, base36enc(backup->start_time), false); - json_add_value(buf, "id", base36enc(backup->start_time), false); + /* Show backup attributes */ + json_add(buf, JT_BEGIN_OBJECT); if (backup->parent_backup != 0) + { json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), true); - - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + base36enc(backup->parent_backup), false); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + } + else + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); @@ -545,6 +562,7 @@ show_instance_json(parray *backup_list) json_add_key(buf, "checksum-version", true); appendPQExpBuffer(buf, "%u", backup->checksum_version); + json_add_value(buf, "program-version", backup->program_version, true); json_add_value(buf, "server-version", backup->server_version, true); json_add_key(buf, "current-tli", true); @@ -594,11 +612,15 @@ show_instance_json(parray *backup_list) json_add_value(buf, "status", status2str(backup->status), true); + json_add(buf, JT_END_OBJECT); + /* End of backup attributes */ + json_add(buf, JT_END_OBJECT); } /* End of backups */ json_add(buf, JT_END_ARRAY); + json_add(buf, JT_END_OBJECT); /* End of instance object */ json_add(buf, JT_END_OBJECT); diff --git a/src/util.c b/src/util.c index 7aeba211..a43239dc 100644 --- a/src/util.c +++ b/src/util.c @@ -325,5 +325,6 @@ pgBackup_init(pgBackup *backup) backup->from_replica = false; backup->parent_backup = 0; backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; } From 8aa559b17b24bc1d91f12df6df2362373fc372dd Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 1 Jun 2018 17:26:56 +0300 Subject: [PATCH 33/99] PGPRO-533: Reformat json output --- src/show.c | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/show.c b/src/show.c index 03ae6488..a39ba528 100644 --- a/src/show.c +++ b/src/show.c @@ -234,9 +234,8 @@ show_instance_start(void) first_instance = true; json_level = 0; - json_add(&show_buf, JT_BEGIN_OBJECT); - json_add_key(&show_buf, "instances", false); - json_add(&show_buf, JT_BEGIN_ARRAY); + appendPQExpBufferChar(&show_buf, '['); + json_level++; } /* @@ -246,11 +245,7 @@ static void show_instance_end(void) { if (show_format == SHOW_JSON) - { - json_add(&show_buf, JT_END_ARRAY); - json_add(&show_buf, JT_END_OBJECT); - appendPQExpBufferChar(&show_buf, '\n'); - } + appendPQExpBufferStr(&show_buf, "\n]\n"); fputs(show_buf.data, stdout); termPQExpBuffer(&show_buf); @@ -505,10 +500,9 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, instance_name, false); - json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, "backups", false); + json_add_value(buf, "instance", instance_name, false); + json_add_key(buf, "backups", true); /* * List backups. @@ -528,19 +522,14 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, base36enc(backup->start_time), false); - /* Show backup attributes */ - json_add(buf, JT_BEGIN_OBJECT); + json_add_value(buf, "id", base36enc(backup->start_time), false); if (backup->parent_backup != 0) - { json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), false); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); - } - else - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + base36enc(backup->parent_backup), true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); @@ -612,15 +601,11 @@ show_instance_json(parray *backup_list) json_add_value(buf, "status", status2str(backup->status), true); - json_add(buf, JT_END_OBJECT); - /* End of backup attributes */ - json_add(buf, JT_END_OBJECT); } /* End of backups */ json_add(buf, JT_END_ARRAY); - json_add(buf, JT_END_OBJECT); /* End of instance object */ json_add(buf, JT_END_OBJECT); From 191d5e30e98d25a70f17ace622b4626f670e9b6d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Jun 2018 20:35:37 +0300 Subject: [PATCH 34/99] tests: json format for show command --- tests/archive.py | 14 +++- tests/backup_test.py | 38 +++++----- tests/delete_test.py | 30 ++++---- tests/delta.py | 4 +- tests/exclude.py | 18 +++-- tests/helpers/ptrack_helpers.py | 121 +++++++++++++++++++------------- tests/pgpro589.py | 2 +- tests/ptrack.py | 103 ++++++++++++++++++--------- tests/restore_test.py | 18 ++--- tests/retention_test.py | 12 ++-- tests/show_test.py | 29 ++++++++ tests/validate_test.py | 84 ++++++++++++---------- 12 files changed, 292 insertions(+), 181 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index c0408360..4e1f39d8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -427,7 +427,11 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): - """make node without archiving, take stream backup and turn it into replica, set replica with archiving, make archive backup from replica""" + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -468,7 +472,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, take FULL backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " @@ -502,7 +508,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, make PAGE backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " diff --git a/tests/backup_test.py b/tests/backup_test.py index 5af59684..1fa74643 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -29,15 +29,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - # full backup mode - # with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - # backup_log.write(self.backup_node(node, options=["--verbose"])) - backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "FULL") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied excluded = True @@ -61,29 +57,29 @@ class BackupTest(ProbackupTest, unittest.TestCase): # print self.show_pb(node) show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PAGE") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") # Check parent backup self.assertEqual( backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # ptrack backup mode self.backup_node(backup_dir, 'node', node, backup_type="ptrack") show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PTRACK") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") # Check parent backup self.assertEqual( page_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -106,7 +102,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") node.stop() # Clean after yourself @@ -162,7 +158,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") # Clean after yourself @@ -227,7 +223,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.assertEqual( self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR") + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") # Clean after yourself self.del_test_dir(module_name, fname) @@ -250,12 +246,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -282,11 +278,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -342,7 +338,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -415,7 +411,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/delete_test.py b/tests/delete_test.py index 039f3606..4afb15ae 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -44,13 +44,13 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node) show_backups = self.show_pb(backup_dir, 'node') - id_1 = show_backups[0]['ID'] - id_2 = show_backups[1]['ID'] - id_3 = show_backups[2]['ID'] + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] self.delete_pb(backup_dir, 'node', id_2) show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(show_backups[0]['ID'], id_1) - self.assertEqual(show_backups[1]['ID'], id_3) + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) # Clean after yourself self.del_test_dir(module_name, fname) @@ -82,15 +82,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -122,15 +122,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index 303c7c58..7cf21758 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1191,7 +1191,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -1264,7 +1264,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/exclude.py b/tests/exclude.py index 763060c7..4f9d73ab 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -34,25 +34,33 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): temp_schema_name = conn.execute("SELECT nspname FROM pg_namespace WHERE oid = pg_my_temp_schema()")[0][0] conn.commit() - temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace("pg_", "") + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") conn.commit() conn.execute("create index test_idx on test (generate_series)") conn.commit() - heap_path = conn.execute("select pg_relation_filepath('test')")[0][0] + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] conn.commit() - index_path = conn.execute("select pg_relation_filepath('test_idx')")[0][0] + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] conn.commit() heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] conn.commit() - toast_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] conn.commit() - toast_idx_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid) + "_index"))[0][0] + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] conn.commit() temp_table_filename = os.path.basename(heap_path) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a776599c..af7fe766 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -12,6 +12,7 @@ import select import psycopg2 from time import sleep import re +import json idx_ptrack = { 't_heap': { @@ -598,7 +599,7 @@ class ProbackupTest(object): def show_pb( self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False + options=[], as_text=False, as_json=True ): backup_list = [] @@ -613,63 +614,83 @@ class ProbackupTest(object): if backup_id: cmd_list += ["-i", backup_id] + if as_json: + cmd_list += ["--format=json"] + if as_text: # You should print it when calling as_text=true return self.run_pb(cmd_list + options) # get show result as list of lines - show_splitted = self.run_pb(cmd_list + options).splitlines() - if instance is not None and backup_id is None: - # cut header(ID, Mode, etc) from show as single string - header = show_splitted[1:2][0] - # cut backup records from show as single list - # with string for every backup record - body = show_splitted[3:] - # inverse list so oldest record come first - body = body[::-1] - # split string in list with string for every header element - header_split = re.split(" +", header) - # Remove empty items - for i in header_split: - if i == '': - header_split.remove(i) + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: continue - header_split = [ - header_element.rstrip() for header_element in header_split - ] - for backup_record in body: - backup_record = backup_record.rstrip() - # split list with str for every backup record element - backup_record_split = re.split(" +", backup_record) - # Remove empty items - for i in backup_record_split: - if i == '': - backup_record_split.remove(i) - if len(header_split) != len(backup_record_split): - print(warning.format( - header=header, body=body, - header_split=header_split, - body_split=backup_record_split) - ) - exit(1) - new_dict = dict(zip(header_split, backup_record_split)) - backup_list.append(new_dict) + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) return backup_list else: - # cut out empty lines and lines started with # - # and other garbage then reconstruct it as dictionary - # print show_splitted - sanitized_show = [item for item in show_splitted if item] - sanitized_show = [ - item for item in sanitized_show if not item.startswith('#') - ] - # print sanitized_show - for line in sanitized_show: - name, var = line.partition(" = ")[::2] - var = var.strip('"') - var = var.strip("'") - specific_record[name.strip()] = var - return specific_record + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record def validate_pb( self, backup_dir, instance=None, diff --git a/tests/pgpro589.py b/tests/pgpro589.py index a67f3dd4..bd40f16d 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -63,7 +63,7 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] self.assertEqual( 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') diff --git a/tests/ptrack.py b/tests/ptrack.py index 9c21ff55..4823acef 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -268,7 +268,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -430,7 +431,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -503,7 +505,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() @@ -584,8 +587,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd) ) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) @@ -604,12 +607,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -688,9 +692,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd) ) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -711,12 +717,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -811,7 +818,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_pgpro417(self): - """Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail""" + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -880,7 +890,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_full_pgpro417(self): - """Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail""" + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -954,7 +967,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_create_db(self): - """Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense""" + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1017,7 +1033,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1046,7 +1063,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1151,7 +1169,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET RESTORED PGDATA AND COMPARE if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1159,8 +1178,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) node_restored.start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node_restored.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1229,7 +1248,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1240,7 +1260,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_drop_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1321,7 +1344,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1379,15 +1405,16 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while restored_node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) # COMPARE LOGICAL CONTENT @@ -1416,14 +1443,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( + while restored_node.psql( "postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) @@ -1437,7 +1465,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_multiple_segments(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1446,9 +1477,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '128MB', - 'maintenance_work_mem': '1GB', 'autovacuum': 'off', - 'full_page_writes': 'off'} + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } ) self.init_pb(backup_dir) @@ -1514,14 +1546,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM NODE_RESTORED if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( + while restored_node.psql( "postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) diff --git a/tests/restore_test.py b/tests/restore_test.py index 4567f37b..fce96911 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -718,7 +718,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # 1 - Try to restore to existing directory node.stop() @@ -785,8 +785,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="page") show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['Status'], "OK") - self.assertEqual(show_pb[2]['Status'], "OK") + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") node.stop() node.cleanup() @@ -829,7 +829,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Full backup self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -845,8 +845,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # First page backup self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: @@ -862,8 +863,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Second page backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") node.stop() node.cleanup() diff --git a/tests/retention_test.py b/tests/retention_test.py index 2d4cac37..652f7c39 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -14,7 +14,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -24,7 +25,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") # Make backups to be purged @@ -57,7 +60,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - #wal_name_b = wal_name.encode('ascii') + # wal_name_b = wal_name.encode('ascii') self.assertEqual(wal_name[8:] > min_wal[8:], True) self.assertEqual(wal_name[8:] > max_wal[8:], True) @@ -68,7 +71,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/show_test.py b/tests/show_test.py index 74bd0341..29d0bdb3 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -36,6 +36,35 @@ class OptionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" diff --git a/tests/validate_test.py b/tests/validate_test.py index 06ea1ea3..afb2305f 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -908,7 +908,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) target_xid = None with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() target_xid = res[0][0] @@ -1041,7 +1042,10 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): - """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), @@ -1083,7 +1087,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): else: walfile = node.safe_psql( 'postgres', - 'select pg_walfile_name(pg_current_wal_location())').rstrip() + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -1134,12 +1138,12 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup STATUS should be "OK"') # Clean after yourself @@ -1208,7 +1212,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Clean after yourself @@ -1405,7 +1409,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: pass self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') self.set_archiving(backup_dir, 'node', node) node.reload() self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1440,14 +1444,19 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') os.rename(file_new, file) try: @@ -1459,14 +1468,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') # Clean after yourself self.del_test_dir(module_name, fname) @@ -1537,13 +1547,13 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') os.rename(file_new, file) file = os.path.join( @@ -1562,13 +1572,13 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') # Clean after yourself self.del_test_dir(module_name, fname) From 93e85cb86e5cc5ba9282fe7d2f2e899682fa0f41 Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Mon, 4 Jun 2018 11:27:00 +0300 Subject: [PATCH 35/99] Add --no-validate option for restore command --- src/pg_probackup.c | 4 +- src/pg_probackup.h | 3 +- src/restore.c | 113 ++++++++++++++++++++++------------------- tests/validate_test.py | 87 +++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 53 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5d464171..598abdec 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -69,6 +69,7 @@ static char *target_action = NULL;; static pgRecoveryTarget *recovery_target_options = NULL; bool restore_as_replica = false; +bool restore_no_validate = false; /* delete options */ bool delete_wal = false; @@ -143,6 +144,7 @@ static pgut_option options[] = { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, /* delete options */ { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, @@ -431,7 +433,7 @@ main(int argc, char *argv[]) /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, target_inclusive, target_tli, target_immediate, - target_name, target_action); + target_name, target_action, restore_no_validate); } if (num_threads < 1) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ce..76d5f3c9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -247,6 +247,7 @@ typedef struct pgRecoveryTarget bool recovery_target_immediate; const char *recovery_target_name; const char *recovery_target_action; + bool restore_no_validate; } pgRecoveryTarget; /* Union to ease operations on relation pages */ @@ -378,7 +379,7 @@ extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, bool target_immediate, - const char *target_name, const char *target_action); + const char *target_name, const char *target_action, bool restore_no_validate); extern void opt_tablespace_map(pgut_option *opt, const char *arg); diff --git a/src/restore.c b/src/restore.c index 69bac841..a5420a6c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -243,66 +243,69 @@ do_restore_or_validate(time_t target_backup_id, if (is_restore) check_tablespace_mapping(dest_backup); - if (dest_backup->backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); - - /* - * Validate backups from base_full_backup to dest_backup. - */ - for (i = base_full_backup_index; i >= dest_backup_index; i--) + if (!is_restore || !rt->restore_no_validate) { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - pgBackupValidate(backup); - /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ - if (backup->status == BACKUP_STATUS_CORRUPT) - { - corrupted_backup = backup; - corrupted_backup_index = i; - break; - } - /* We do not validate WAL files of intermediate backups - * It`s done to speed up restore - */ - } - /* There is no point in wal validation - * if there is corrupted backup between base_backup and dest_backup - */ - if (!corrupted_backup) + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + /* - * Validate corresponding WAL files. - * We pass base_full_backup timeline as last argument to this function, - * because it's needed to form the name of xlog file. + * Validate backups from base_full_backup to dest_backup. */ - validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, base_full_backup->tli); - - /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ - if (corrupted_backup) - { - for (i = corrupted_backup_index - 1; i >= 0; i--) + for (i = base_full_backup_index; i >= dest_backup_index; i--) { pgBackup *backup = (pgBackup *) parray_get(backups, i); - /* Mark incremental OK backup as orphan */ - if (backup->backup_mode == BACKUP_MODE_FULL) - break; - if (backup->status != BACKUP_STATUS_OK) - continue; - else + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) { - char *backup_id, - *corrupted_backup_id; + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, base_full_backup->tli); - backup->status = BACKUP_STATUS_ORPHAN; - pgBackupWriteBackupControlFile(backup); + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; - backup_id = base36enc_dup(backup->start_time); - corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); - elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", - backup_id, corrupted_backup_id); + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); - free(backup_id); - free(corrupted_backup_id); + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } } } } @@ -1001,7 +1004,8 @@ parseRecoveryTargetOptions(const char *target_time, TimeLineID target_tli, bool target_immediate, const char *target_name, - const char *target_action) + const char *target_action, + bool restore_no_validate) { time_t dummy_time; TransactionId dummy_xid; @@ -1026,6 +1030,7 @@ parseRecoveryTargetOptions(const char *target_time, rt->recovery_target_immediate = false; rt->recovery_target_name = NULL; rt->recovery_target_action = NULL; + rt->restore_no_validate = false; /* parse given options */ if (target_time) @@ -1072,6 +1077,12 @@ parseRecoveryTargetOptions(const char *target_time, rt->recovery_target_immediate = target_immediate; } + if (restore_no_validate) + { + recovery_target_specified++; + rt->restore_no_validate = restore_no_validate; + } + if (target_name) { recovery_target_specified++; diff --git a/tests/validate_test.py b/tests/validate_test.py index 06ea1ea3..2cebcc33 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -748,6 +748,93 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_validate_instance_with_corrupted_full(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, From 02fd1878bb60fb87baa219f6a07f8c71d89041e8 Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Tue, 5 Jun 2018 16:06:22 +0300 Subject: [PATCH 36/99] Add help message for --no-validate --- src/help.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index 2f84b225..70d8b4d5 100644 --- a/src/help.c +++ b/src/help.c @@ -265,7 +265,7 @@ help_restore(void) printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica]\n\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -282,6 +282,7 @@ help_restore(void) printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --no-validate disable backup validation during recovery\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --recovery-target-action=pause|promote|shutdown\n")); From c71151d3df99b5b656190da7225e48ab82c01d7c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Jun 2018 14:07:10 +0300 Subject: [PATCH 37/99] PGPRO-533: Add json format for show-config --- Makefile | 3 +- src/configure.c | 162 ++++++++++++++++++++++++++++++++- src/show.c | 232 ++++++++++++----------------------------------- src/utils/json.c | 134 +++++++++++++++++++++++++++ src/utils/json.h | 33 +++++++ 5 files changed, 388 insertions(+), 176 deletions(-) create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h diff --git a/Makefile b/Makefile index 0f4fa672..9880e5db 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/pg_probackup.o src/restore.o src/show.o src/status.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ - src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h diff --git a/src/configure.c b/src/configure.c index 55d2bbc5..0410dc74 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,19 +2,33 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" +#include "pqexpbuffer.h" + +#include "utils/json.h" + + static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + static pgBackupConfig *cur_config = NULL; +static PQExpBufferData show_buf; +static int32 json_level = 0; + /* Set configure options */ int do_configure(bool show_only) @@ -68,7 +82,7 @@ do_configure(bool show_only) config->compress_level = compress_level; if (show_only) - writeBackupCatalogConfig(stderr, config); + show_configure(config); else writeBackupCatalogConfigFile(config); @@ -251,7 +265,6 @@ readBackupCatalogConfigFile(void) pgut_readopt(path, options, ERROR); return config; - } static void @@ -271,3 +284,146 @@ opt_compress_alg(pgut_option *opt, const char *arg) { cur_config->compress_alg = parse_compress_alg(arg); } + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + if (config->replica_timeout != INT_MIN) + { + json_add_key(buf, "replica-timeout", json_level, true); + appendPQExpBuffer(buf, "%d", config->replica_timeout); + } + + /* Logging parameters */ + if (config->log_level_console != INT_MIN) + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + if (config->log_level_file != INT_MIN) + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + if (config->log_filename) + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + if (config->log_directory) + json_add_value(buf, "log-directory", config->log_directory, json_level, + true); + + if (config->log_rotation_size) + { + json_add_key(buf, "log-rotation-size", json_level, true); + appendPQExpBuffer(buf, "%d", config->log_rotation_size); + } + if (config->log_rotation_age) + { + json_add_key(buf, "log-rotation-age", json_level, true); + appendPQExpBuffer(buf, "%d", config->log_rotation_age); + } + + /* Retention parameters */ + if (config->retention_redundancy) + { + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + } + if (config->retention_window) + { + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + } + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/show.c b/src/show.c index a39ba528..01a558e0 100644 --- a/src/show.c +++ b/src/show.c @@ -17,6 +17,8 @@ #include "pqexpbuffer.h" +#include "utils/json.h" + static void show_instance_start(void); static void show_instance_end(void); @@ -26,24 +28,9 @@ static int show_backup(time_t requested_backup_id); static void show_instance_plain(parray *backup_list, bool show_name); static void show_instance_json(parray *backup_list); -/* Json output functions */ - -typedef enum -{ - JT_BEGIN_ARRAY, - JT_END_ARRAY, - JT_BEGIN_OBJECT, - JT_END_OBJECT -} JsonToken; - -static void json_add(PQExpBuffer buf, JsonToken type); -static void json_add_key(PQExpBuffer buf, const char *name, bool add_comma); -static void json_add_value(PQExpBuffer buf, const char *name, const char *value, - bool add_comma); - static PQExpBufferData show_buf; static bool first_instance = true; -static uint8 json_level = 0; +static int32 json_level = 0; int do_show(time_t requested_backup_id) @@ -377,115 +364,6 @@ show_instance_plain(parray *backup_list, bool show_name) * Json output. */ -static void -json_add_indent(PQExpBuffer buf) -{ - uint8 i; - - if (json_level == 0) - return; - - appendPQExpBufferChar(buf, '\n'); - for (i = 0; i < json_level; i++) - appendPQExpBufferStr(buf, " "); -} - -static void -json_add(PQExpBuffer buf, JsonToken type) -{ - switch (type) - { - case JT_BEGIN_ARRAY: - appendPQExpBufferChar(buf, '['); - json_level++; - break; - case JT_END_ARRAY: - json_level--; - if (json_level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf); - appendPQExpBufferChar(buf, ']'); - break; - case JT_BEGIN_OBJECT: - json_add_indent(buf); - appendPQExpBufferChar(buf, '{'); - json_level++; - break; - case JT_END_OBJECT: - json_level--; - if (json_level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf); - appendPQExpBufferChar(buf, '}'); - break; - default: - break; - } -} - -static void -json_add_escaped(PQExpBuffer buf, const char *str) -{ - const char *p; - - appendPQExpBufferChar(buf, '"'); - for (p = str; *p; p++) - { - switch (*p) - { - case '\b': - appendPQExpBufferStr(buf, "\\b"); - break; - case '\f': - appendPQExpBufferStr(buf, "\\f"); - break; - case '\n': - appendPQExpBufferStr(buf, "\\n"); - break; - case '\r': - appendPQExpBufferStr(buf, "\\r"); - break; - case '\t': - appendPQExpBufferStr(buf, "\\t"); - break; - case '"': - appendPQExpBufferStr(buf, "\\\""); - break; - case '\\': - appendPQExpBufferStr(buf, "\\\\"); - break; - default: - if ((unsigned char) *p < ' ') - appendPQExpBuffer(buf, "\\u%04x", (int) *p); - else - appendPQExpBufferChar(buf, *p); - break; - } - } - appendPQExpBufferChar(buf, '"'); -} - -static void -json_add_key(PQExpBuffer buf, const char *name, bool add_comma) -{ - if (add_comma) - appendPQExpBufferChar(buf, ','); - json_add_indent(buf); - - json_add_escaped(buf, name); - appendPQExpBufferStr(buf, ": "); -} - -static void -json_add_value(PQExpBuffer buf, const char *name, const char *value, - bool add_comma) -{ - json_add_key(buf, name, add_comma); - json_add_escaped(buf, value); -} - /* * Show instance backups in json format. */ @@ -499,116 +377,126 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); /* Begin of instance object */ - json_add(buf, JT_BEGIN_OBJECT); + json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "instance", instance_name, false); - json_add_key(buf, "backups", true); + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); /* * List backups. */ - json_add(buf, JT_BEGIN_ARRAY); + json_add(buf, JT_BEGIN_ARRAY, &json_level); for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = parray_get(backup_list, i); TimeLineID parent_tli; char timestamp[100] = "----"; - char duration[20] = "----"; - char data_bytes_str[10] = "----"; char lsn[20]; if (i != 0) appendPQExpBufferChar(buf, ','); - json_add(buf, JT_BEGIN_OBJECT); + json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "id", base36enc(backup->start_time), false); + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), true); + base36enc(backup->parent_backup), json_level, true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); - json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); json_add_value(buf, "compress-alg", - deparse_compress_alg(backup->compress_alg), true); + deparse_compress_alg(backup->compress_alg), json_level, + true); - json_add_key(buf, "compress-level", true); + json_add_key(buf, "compress-level", json_level, true); appendPQExpBuffer(buf, "%d", backup->compress_level); json_add_value(buf, "from-replica", - backup->from_replica ? "true" : "false", true); + backup->from_replica ? "true" : "false", json_level, + true); - json_add_key(buf, "block-size", true); + json_add_key(buf, "block-size", json_level, true); appendPQExpBuffer(buf, "%u", backup->block_size); - json_add_key(buf, "xlog-block-size", true); + json_add_key(buf, "xlog-block-size", json_level, true); appendPQExpBuffer(buf, "%u", backup->wal_block_size); - json_add_key(buf, "checksum-version", true); + json_add_key(buf, "checksum-version", json_level, true); appendPQExpBuffer(buf, "%u", backup->checksum_version); - json_add_value(buf, "program-version", backup->program_version, true); - json_add_value(buf, "server-version", backup->server_version, true); + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); - json_add_key(buf, "current-tli", true); + json_add_key(buf, "current-tli", json_level, true); appendPQExpBuffer(buf, "%d", backup->tli); - json_add_key(buf, "parent-tli", true); + json_add_key(buf, "parent-tli", json_level, true); parent_tli = get_parent_tli(backup->tli); appendPQExpBuffer(buf, "%u", parent_tli); snprintf(lsn, lengthof(lsn), "%X/%X", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); - json_add_value(buf, "start-lsn", lsn, true); + json_add_value(buf, "start-lsn", lsn, json_level, true); snprintf(lsn, lengthof(lsn), "%X/%X", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - json_add_value(buf, "stop-lsn", lsn, true); + json_add_value(buf, "stop-lsn", lsn, json_level, true); time2iso(timestamp, lengthof(timestamp), backup->start_time); - json_add_value(buf, "start-time", timestamp, true); + json_add_value(buf, "start-time", timestamp, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->end_time); - json_add_value(buf, "end-time", timestamp, true); + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } - json_add_key(buf, "recovery-xid", true); + json_add_key(buf, "recovery-xid", json_level, true); appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - json_add_value(buf, "recovery-time", timestamp, true); - - pretty_size(backup->data_bytes, data_bytes_str, - lengthof(data_bytes_str)); - json_add_value(buf, "data-bytes", data_bytes_str, true); - - pretty_size(backup->wal_bytes, data_bytes_str, - lengthof(data_bytes_str)); - json_add_value(buf, "wal-bytes", data_bytes_str, true); - - if (backup->end_time != (time_t) 0) + if (backup->recovery_time > 0) { - snprintf(duration, lengthof(duration), "%.*lfs", 0, - difftime(backup->end_time, backup->start_time)); - json_add_value(buf, "time", duration, true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); } if (backup->primary_conninfo) - json_add_value(buf, "primary_conninfo", backup->primary_conninfo, true); + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); - json_add_value(buf, "status", status2str(backup->status), true); + json_add_value(buf, "status", status2str(backup->status), json_level, + true); - json_add(buf, JT_END_OBJECT); + json_add(buf, JT_END_OBJECT, &json_level); } /* End of backups */ - json_add(buf, JT_END_ARRAY); + json_add(buf, JT_END_ARRAY, &json_level); /* End of instance object */ - json_add(buf, JT_END_OBJECT); + json_add(buf, JT_END_OBJECT, &json_level); first_instance = false; } diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 00000000..3afbe9e7 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 00000000..cf5a7064 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ From efc0c8cda31dc58c43379b018136e0c0c7b458ae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 Jun 2018 12:22:45 +0300 Subject: [PATCH 38/99] bugfix: WAL delivery timeout could cause PostgreSQL instance to be left in in_backup state --- src/backup.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index a0b02fcf..f0f7ac9c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1076,6 +1076,12 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) 2, params); + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ @@ -1108,12 +1114,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* Wait for start_lsn to be replayed by replica */ if (from_replica) wait_replica_wal_lsn(backup->start_lsn, true); - - /* - * Set flag that pg_start_backup() was called. If an error will happen it - * is necessary to call pg_stop_backup() in backup_cleanup(). - */ - backup_in_progress = true; } /* From 9cd12fbe055b0bf1e494357713b88d2c1c622680 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 7 Jun 2018 14:54:25 +0300 Subject: [PATCH 39/99] PGPRO-533: Update hellp command --- src/configure.c | 4 ++++ src/help.c | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/configure.c b/src/configure.c index 0410dc74..dc80981a 100644 --- a/src/configure.c +++ b/src/configure.c @@ -29,6 +29,10 @@ static pgBackupConfig *cur_config = NULL; static PQExpBufferData show_buf; static int32 json_level = 0; +/* + * All this code needs refactoring. + */ + /* Set configure options */ int do_configure(bool show_only) diff --git a/src/help.c b/src/help.c index a41e8167..28500e75 100644 --- a/src/help.c +++ b/src/help.c @@ -89,6 +89,7 @@ help_pg_probackup(void) printf(_(" [--replica-timeout=timeout]\n")); printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); @@ -358,7 +359,8 @@ static void help_show(void) { printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n\n")); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); @@ -474,10 +476,12 @@ help_set_config(void) static void help_show_config(void) { - printf(_("%s show-config -B backup-dir --instance=instance_name\n\n"), PROGRAM_NAME); + printf(_("%s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void From e36daf5d1d8d42cd5413715e60b67328e701177b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 7 Jun 2018 19:13:11 +0300 Subject: [PATCH 40/99] Merge master into pg_probackup_windows --- Makefile | 3 +- doit.cmd | 1 + doit96.cmd | 1 + msvs/pg_probackup.sln | 28 ++++ msvs/template.pg_probackup.vcxproj | 209 +++++++++++++++++++++++ msvs/template.pg_probackup96.vcxproj | 207 +++++++++++++++++++++++ msvs/template.pg_probackup_2.vcxproj | 203 ++++++++++++++++++++++ src/backup.c | 73 ++++---- src/catalog.c | 1 - src/configure.c | 3 +- src/data.c | 56 +++---- src/dir.c | 18 +- src/pg_probackup.c | 15 +- src/pg_probackup.h | 27 ++- src/restore.c | 72 ++++---- src/status.c | 2 +- src/utils/logger.c | 22 ++- src/utils/pgut.c | 12 +- src/utils/pgut.h | 8 - src/utils/thread.c | 80 +++++++++ src/utils/thread.h | 32 ++++ src/validate.c | 44 +++-- win32build.pl | 240 +++++++++++++++++++++++++++ win32build96.pl | 240 +++++++++++++++++++++++++++ win32build_2.pl | 219 ++++++++++++++++++++++++ 25 files changed, 1664 insertions(+), 152 deletions(-) create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/Makefile b/Makefile index 9880e5db..eb3e3ee2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ - src/utils/json.o + src/utils/json.o src/utils/thread.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h @@ -63,6 +63,7 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..cc29bc88 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..044a3e73 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,209 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..c095d7cc --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,207 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..2fc101a4 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 5b4063fa..f24a0b0f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -18,16 +18,16 @@ #include #include #include -#include -#include "libpq/pqsignal.h" -#include "storage/bufpage.h" #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" #include "datapagemap.h" -#include "receivelog.h" -#include "streamutil.h" +#include "libpq/pqsignal.h" #include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; @@ -89,8 +89,8 @@ static bool pg_stop_backup_is_sent = false; static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -static void backup_files(void *arg); -static void remote_backup_files(void *arg); +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); static void do_backup_instance(void); @@ -253,7 +253,11 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) else if (copybuf[156] == '2') { /* Symlink */ +#ifndef WIN32 pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif } else elog(ERROR, "Unrecognized link indicator \"%c\"\n", @@ -289,7 +293,7 @@ remote_copy_file(PGconn *conn, pgFile* file) DATABASE_DIR); join_path_components(to_path, database_path, file->path); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -363,7 +367,7 @@ remote_copy_file(PGconn *conn, pgFile* file) * Take a remote backup of the PGDATA at a file level. * Copy all directories and files listed in backup_files_list. */ -static void +static void * remote_backup_files(void *arg) { int i; @@ -385,7 +389,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); @@ -441,6 +445,8 @@ remote_backup_files(void *arg) /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* @@ -456,8 +462,9 @@ do_backup_instance(void) char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - pthread_t backup_threads[num_threads]; - backup_files_args *backup_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *backup_threads; + backup_files_args *backup_threads_args; bool backup_isok = true; pgBackup *prev_backup = NULL; @@ -681,16 +688,19 @@ do_backup_instance(void) } /* setup threads */ - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); /* init thread args with own file lists */ + backup_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + backup_threads_args = (backup_files_args *) palloc(sizeof(backup_files_args)*num_threads); + for (i = 0; i < num_threads; i++) { - backup_files_args *arg = pg_malloc(sizeof(backup_files_args)); + backup_files_args *arg = &(backup_threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; @@ -701,33 +711,27 @@ do_backup_instance(void) arg->thread_cancel_conn = NULL; /* By default there are some error */ arg->ret = 1; - backup_threads_args[i] = arg; } /* Run threads */ elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { + backup_files_args *arg = &(backup_threads_args[i]); elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) backup_files, - backup_threads_args[i]); + pthread_create(&backup_threads[i], NULL, backup_files, arg); else - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) remote_backup_files, - backup_threads_args[i]); + pthread_create(&backup_threads[i], NULL, remote_backup_files, arg); } - + /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); - if (backup_threads_args[i]->ret == 1) + if (backup_threads_args[i].ret == 1) backup_isok = false; - - pg_free(backup_threads_args[i]); } if (backup_isok) elog(LOG, "Data files are transfered"); @@ -1017,7 +1021,7 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - + if (system_id_conn != system_identifier) elog(ERROR, "Backup data directory was initialized for system id %ld, but connected instance system id is %ld", system_identifier, system_id_conn); @@ -1040,14 +1044,15 @@ confirm_block_size(const char *name, int blcksz) res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); - + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - PQclear(res); - if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); + + PQclear(res);//bad pointer to endp + } /* @@ -1802,7 +1807,7 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, "w"); + fp = fopen(backup_label, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1850,7 +1855,7 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, "w"); + fp = fopen(tablespace_map, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); @@ -2006,7 +2011,7 @@ backup_disconnect(bool fatal, void *userdata) * In incremental backup mode, copy only files or datafiles' pages changed after * previous backup. */ -static void +static void * backup_files(void *arg) { int i; @@ -2021,7 +2026,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ @@ -2126,6 +2131,8 @@ backup_files(void *arg) /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* diff --git a/src/catalog.c b/src/catalog.c index eb5d3be7..f19628c3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/src/configure.c b/src/configure.c index dc80981a..2fc27892 100644 --- a/src/configure.c +++ b/src/configure.c @@ -132,7 +132,8 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); + //fprintf(out, "system-identifier = %li\n", config->system_identifier); + fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/data.c b/src/data.c index a1bdb91a..be07ce64 100644 --- a/src/data.c +++ b/src/data.c @@ -446,7 +446,7 @@ backup_data_file(backup_files_args* arguments, INIT_CRC32C(file->crc); /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(file->crc); @@ -480,7 +480,7 @@ backup_data_file(backup_files_args* arguments, /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -569,18 +569,18 @@ restore_data_file(const char *from_root, pgFile *file, pgBackup *backup) { - char to_path[MAXPGPATH]; - FILE *in = NULL; - FILE *out = NULL; - BackupPageHeader header; - BlockNumber blknum; - size_t file_size; + char to_path[MAXPGPATH]; + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum; + size_t file_size; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) { /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { elog(ERROR, "cannot open backup file \"%s\": %s", file->path, @@ -594,9 +594,9 @@ restore_data_file(const char *from_root, * re-open it with "w" to create an empty file. */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "r+"); + out = fopen(to_path, PG_BINARY_R "+"); if (out == NULL && errno == ENOENT) - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -690,13 +690,13 @@ restore_data_file(const char *from_root, } } - /* - * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do - * But during DELTA backup we read every file in PGDATA and thus DELTA backup - * knows exact size of every file at the time of backup. - * So when restoring file from DELTA backup we, knowning it`s size at - * a time of a backup, can truncate file to this size. - */ + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA) { @@ -759,7 +759,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); @@ -775,7 +775,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -918,7 +918,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out; + FILE *out=NULL; char buf[XLOG_BLCKSZ]; const char *to_path_p = to_path; char to_path_temp[MAXPGPATH]; @@ -930,7 +930,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -946,7 +946,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = gzopen(to_path_temp, "wb"); + gz_out = gzopen(to_path_temp, PG_BINARY_W); if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", compress_level, to_path_temp, get_gz_error(gz_out, errno)); @@ -961,7 +961,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1083,7 +1083,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) { #ifdef HAVE_LIBZ @@ -1092,7 +1092,7 @@ get_wal_file(const char *from_path, const char *to_path) * extension. */ snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - gz_in = gzopen(gz_from_path, "rb"); + gz_in = gzopen(gz_from_path, PG_BINARY_R); if (gz_in == NULL) { if (errno == ENOENT) @@ -1120,7 +1120,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1254,7 +1254,7 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); diff --git a/src/dir.c b/src/dir.c index 8df3da3f..20725f88 100644 --- a/src/dir.c +++ b/src/dir.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include #include #include @@ -102,10 +101,12 @@ int dir_create_dir(const char *dir, mode_t mode) { char copy[MAXPGPATH]; - char parent[MAXPGPATH]; + char *parent; strncpy(copy, dir, MAXPGPATH); - strncpy(parent, dirname(copy), MAXPGPATH); + + parent = pstrdup(dir); + get_parent_directory(parent); /* Create parent first */ if (access(parent, F_OK) == -1) @@ -119,6 +120,7 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } + pfree(parent); return 0; } @@ -153,6 +155,8 @@ pgFileInit(const char *path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); + file->name = 0; + file->size = 0; file->mode = 0; file->read_size = 0; @@ -232,7 +236,7 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, "r"); + fp = fopen(file->path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", file->path, strerror(errno)); @@ -350,7 +354,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, "r"); + black_list_file = fopen(path, PG_BINARY_R); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); @@ -827,7 +831,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); +#ifndef WIN32 if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != -1) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index efb8e6a3..c66179e1 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -10,6 +10,7 @@ #include "pg_probackup.h" #include "streamutil.h" +#include "utils/thread.h" #include #include @@ -207,11 +208,7 @@ main(int argc, char *argv[]) /* * Save main thread's tid. It is used call exit() in case of errors. */ -#ifdef WIN32 - main_tid = GetCurrentThreadId(); -#else main_tid = pthread_self(); -#endif /* Parse subcommands and non-subcommand options */ if (argc > 1) @@ -235,7 +232,7 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "show") == 0) backup_subcmd = SHOW; else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; + backup_subcmd = DELETE_SUBCMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG; else if (strcmp(argv[1], "show-config") == 0) @@ -281,7 +278,7 @@ main(int argc, char *argv[]) if (backup_subcmd == BACKUP || backup_subcmd == RESTORE || backup_subcmd == VALIDATE || - backup_subcmd == DELETE) + backup_subcmd == DELETE_SUBCMD) { int i, len = 0, @@ -325,6 +322,7 @@ main(int argc, char *argv[]) if (backup_path == NULL) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } + canonicalize_path(backup_path); /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) @@ -402,7 +400,7 @@ main(int argc, char *argv[]) { if (backup_subcmd != RESTORE && backup_subcmd != VALIDATE - && backup_subcmd != DELETE + && backup_subcmd != DELETE_SUBCMD && backup_subcmd != SHOW) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -464,6 +462,7 @@ main(int argc, char *argv[]) start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, @@ -484,7 +483,7 @@ main(int argc, char *argv[]) false); case SHOW: return do_show(current.backup_id); - case DELETE: + case DELETE_SUBCMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d3ce8241..8face84d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -15,10 +15,6 @@ #include #include -#ifndef WIN32 -#include -#endif - #include "access/timeline.h" #include "access/xlogdefs.h" #include "access/xlog_internal.h" @@ -28,6 +24,13 @@ #include "storage/checksum.h" #include "utils/pg_crc.h" #include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif #include "utils/parray.h" #include "utils/pgut.h" @@ -102,7 +105,7 @@ typedef struct pgFile bool is_database; bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile uint32 lock; /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; @@ -144,7 +147,7 @@ typedef enum ProbackupSubcmd RESTORE, VALIDATE, SHOW, - DELETE, + DELETE_SUBCMD, SET_CONFIG, SHOW_CONFIG } ProbackupSubcmd; @@ -531,4 +534,16 @@ extern void pgBackup_init(pgBackup *backup); /* in status.c */ extern bool is_pg_running(void); +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + #endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c index 69bac841..0e42c4c4 100644 --- a/src/restore.c +++ b/src/restore.c @@ -14,14 +14,14 @@ #include #include #include -#include #include "catalog/pg_control.h" +#include "utils/thread.h" typedef struct { - parray *files; - pgBackup *backup; + parray *files; + pgBackup *backup; /* * Return value from the thread. @@ -65,7 +65,7 @@ static void check_tablespace_mapping(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); -static void restore_files(void *arg); +static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); static const char *get_tablespace_mapping(const char *dir); static void set_tablespace_created(const char *link, const char *dir); @@ -80,13 +80,11 @@ static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int -do_restore_or_validate(time_t target_backup_id, - pgRecoveryTarget *rt, - bool is_restore) +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) { int i; parray *backups; - parray *timelines; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; @@ -169,6 +167,8 @@ do_restore_or_validate(time_t target_backup_id, if (rt->recovery_target_tli) { + parray *timelines; + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); /* Read timeline history files from archives */ timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); @@ -362,8 +362,9 @@ restore_backup(pgBackup *backup) char list_path[MAXPGPATH]; parray *files; int i; - pthread_t restore_threads[num_threads]; - restore_files_args *restore_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *restore_threads; + restore_files_args *restore_threads_args; bool restore_isok = true; if (backup->status != BACKUP_STATUS_OK) @@ -397,17 +398,21 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); files = dir_read_file_list(database_path, list_path); + restore_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + restore_threads_args = (restore_files_args *) palloc(sizeof(restore_files_args)*num_threads); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = pg_malloc(sizeof(restore_files_args)); + restore_files_args *arg = &(restore_threads_args[i]); + arg->files = files; arg->backup = backup; /* By default there are some error */ @@ -415,23 +420,22 @@ restore_backup(pgBackup *backup) elog(LOG, "Start thread for num:%li", parray_num(files)); - restore_threads_args[i] = arg; - pthread_create(&restore_threads[i], NULL, - (void *(*)(void *)) restore_files, arg); + pthread_create(&restore_threads[i], NULL, restore_files, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { pthread_join(restore_threads[i], NULL); - if (restore_threads_args[i]->ret == 1) + if (restore_threads_args[i].ret == 1) restore_isok = false; - - pg_free(restore_threads_args[i]); } if (!restore_isok) elog(ERROR, "Data files restoring failed"); + pfree(restore_threads); + pfree(restore_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); @@ -452,7 +456,7 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - int i; + int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ @@ -702,7 +706,7 @@ check_tablespace_mapping(pgBackup *backup) /* * Restore files into $PGDATA. */ -static void +static void * restore_files(void *arg) { int i; @@ -714,7 +718,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, @@ -779,6 +783,8 @@ restore_files(void *arg) /* Data files restoring is successful */ arguments->ret = 0; + + return NULL; } /* Create recovery.conf with given recovery target parameters */ @@ -787,9 +793,9 @@ create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup) { - char path[MAXPGPATH]; - FILE *fp; - bool need_restore_conf = false; + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; if (!backup->stream || (rt->time_specified || rt->xid_specified)) @@ -959,7 +965,8 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - entry->end = (uint32) (-1UL << 32) | -1UL; + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; parray_insert(result, 0, entry); return result; @@ -980,10 +987,13 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) bool satisfy_timeline(const parray *timelines, const pgBackup *backup) { - int i; + int i; + for (i = 0; i < parray_num(timelines); i++) { - TimeLineHistoryEntry *timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && backup->stop_lsn < timeline->end) return true; @@ -1003,14 +1013,14 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_name, const char *target_action) { - time_t dummy_time; - TransactionId dummy_xid; - bool dummy_bool; + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; /* * count the number of the mutually exclusive options which may specify * recovery target. If final value > 1, throw an error. */ - int recovery_target_specified = 0; + int recovery_target_specified = 0; pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); /* fill all options with default values */ diff --git a/src/status.c b/src/status.c index 1c7c6038..155a07f4 100644 --- a/src/status.c +++ b/src/status.c @@ -38,7 +38,7 @@ get_pgpid(void) snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); - pidf = fopen(pid_file, "r"); + pidf = fopen(pid_file, PG_BINARY_R); if (pidf == NULL) { /* No pid file, not an error on startup */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 0a4f835b..8fd78739 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -16,6 +15,7 @@ #include "logger.h" #include "pgut.h" +#include "thread.h" /* Logger parameters */ @@ -69,6 +69,9 @@ static bool exit_hook_registered = false; static bool loggin_in_progress = false; static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; +#ifdef WIN32 +static long mutex_initlock = 0; +#endif void init_logger(const char *root_path) @@ -138,11 +141,10 @@ exit_if_necessary(int elevel) } /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) #ifdef WIN32 - if (main_tid != GetCurrentThreadId()) ExitThread(elevel); #else - if (!pthread_equal(main_tid, pthread_self())) pthread_exit(NULL); #endif else @@ -174,6 +176,16 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) /* * There is no need to lock if this is elog() from upper elog(). */ +#ifdef WIN32 + if (log_file_mutex == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (log_file_mutex == NULL) + pthread_mutex_init(&log_file_mutex, NULL); + InterlockedExchange(&mutex_initlock, 0); + } +#endif pthread_mutex_lock(&log_file_mutex); loggin_in_progress = true; @@ -550,7 +562,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, "r"); + control_file = fopen(control, PG_BINARY_R); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -596,7 +608,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, "w"); + control_file = fopen(control, PG_BINARY_W); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 56263cfa..1a7aae0b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -42,12 +42,6 @@ static char *password = NULL; bool prompt_password = true; bool force_password = false; -#ifdef WIN32 -DWORD main_tid = 0; -#else -pthread_t main_tid = 0; -#endif - /* Database connections */ static PGcancel *volatile cancel_conn = NULL; @@ -995,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) + if (!isprint(opts[i].val)) //opts[i].val > 128 || continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1052,6 +1046,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; + len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1059,6 +1054,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ + optind = 2; while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); @@ -1225,7 +1221,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "# \n\r\t\v"); + i = j = strcspn(s, "#\n\r\t\v");//removed space memcpy(dst, s, j); } diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 803d2c57..a27cea83 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -15,7 +15,6 @@ #include "pqexpbuffer.h" #include -#include #include #include "logger.h" @@ -94,13 +93,6 @@ extern const char *PROGRAM_VERSION; extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; -/* ID of the main thread */ -#ifdef WIN32 -extern DWORD main_tid; -#else -extern pthread_t main_tid; -#endif - extern void pgut_help(bool details); /* diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 00000000..da3544a1 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 00000000..0a113d58 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,32 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c index afc7f07d..cb693268 100644 --- a/src/validate.c +++ b/src/validate.c @@ -11,10 +11,11 @@ #include "pg_probackup.h" #include -#include #include -static void pgBackupValidateFiles(void *arg); +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; @@ -42,8 +43,9 @@ pgBackupValidate(pgBackup *backup) parray *files; bool corrupted = false; bool validation_isok = true; - pthread_t validate_threads[num_threads]; - validate_files_args *validate_threads_args[num_threads]; + /* arrays with meta info for multi threaded validate */ + pthread_t *validate_threads; + validate_files_args *validate_threads_args; int i; /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ @@ -77,36 +79,44 @@ pgBackupValidate(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } + /* init thread args with own file lists */ + validate_threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + validate_threads_args = (validate_files_args *) + palloc(sizeof(validate_files_args) * num_threads); + /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = pg_malloc(sizeof(validate_files_args)); + validate_files_args *arg = &(validate_threads_args[i]); + arg->files = files; arg->corrupted = false; /* By default there are some error */ arg->ret = 1; - validate_threads_args[i] = arg; - pthread_create(&validate_threads[i], NULL, - (void *(*)(void *)) pgBackupValidateFiles, arg); + pthread_create(&validate_threads[i], NULL, pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { + validate_files_args *arg = &(validate_threads_args[i]); + pthread_join(validate_threads[i], NULL); - if (validate_threads_args[i]->corrupted) + if (arg->corrupted) corrupted = true; - if (validate_threads_args[i]->ret == 1) + if (arg->ret == 1) validation_isok = false; - pg_free(validate_threads_args[i]); } if (!validation_isok) elog(ERROR, "Data files validation failed"); + pfree(validate_threads); + pfree(validate_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); @@ -127,10 +137,10 @@ pgBackupValidate(pgBackup *backup) * rather throw a WARNING and set arguments->corrupted = true. * This is necessary to update backup status. */ -static void +static void * pgBackupValidateFiles(void *arg) { - int i; + int i; validate_files_args *arguments = (validate_files_args *)arg; pg_crc32 crc; @@ -139,7 +149,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) @@ -198,6 +208,8 @@ pgBackupValidateFiles(void *arg) /* Data files validation is successful */ arguments->ret = 0; + + return NULL; } /* @@ -267,7 +279,7 @@ do_validate_all(void) static void do_validate_instance(void) { - char *current_backup_id; + char *current_backup_id; int i; parray *backups; pgBackup *current_backup = NULL; diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From bd1f5c478c0c0bbd554f86fd234051c3a0eb18e3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Fri, 8 Jun 2018 13:22:39 +0300 Subject: [PATCH 41/99] Some changes after Windows testing --- doit.cmd | 2 +- msvs/template.pg_probackup.vcxproj | 4 ++++ msvs/template.pg_probackup96.vcxproj | 4 ++++ src/utils/thread.c | 1 + src/utils/thread.h | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doit.cmd b/doit.cmd index cc29bc88..b46e3b36 100644 --- a/doit.cmd +++ b/doit.cmd @@ -1 +1 @@ -perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 044a3e73..9b1d84dd 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -191,6 +191,8 @@ + + @@ -202,6 +204,8 @@ + + diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj index c095d7cc..185c6382 100644 --- a/msvs/template.pg_probackup96.vcxproj +++ b/msvs/template.pg_probackup96.vcxproj @@ -190,6 +190,8 @@ + + @@ -200,6 +202,8 @@ + + diff --git a/src/utils/thread.c b/src/utils/thread.c index da3544a1..a12a1c83 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -12,6 +12,7 @@ pthread_t main_tid = 0; #ifdef WIN32 +#include typedef struct win32_pthread { diff --git a/src/utils/thread.h b/src/utils/thread.h index 0a113d58..1a50f1da 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -11,6 +11,7 @@ #define PROBACKUP_THREAD_H #ifdef WIN32 +#include "postgres_fe.h" #include "port/pthread-win32.h" /* Use native win32 threads on Windows */ From 74d9e437749f0fb3f99aab3483e499f2d92bf05b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 12:32:40 +0300 Subject: [PATCH 42/99] PGPRO-1646: Use int for write_size instead of size_t --- src/backup.c | 33 +++++++++++++++------------------ src/data.c | 8 ++++---- src/dir.c | 8 ++++---- src/pg_probackup.h | 4 ++-- src/restore.c | 4 ++-- src/validate.c | 5 ++--- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/backup.c b/src/backup.c index f24a0b0f..c96d8f17 100644 --- a/src/backup.c +++ b/src/backup.c @@ -357,7 +357,7 @@ remote_copy_file(PGconn *conn, pgFile* file) elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); } - file->write_size = file->read_size; + file->write_size = (int) file->read_size; FIN_CRC32C(file->crc); fclose(out); @@ -438,8 +438,8 @@ remote_backup_files(void *arg) /* receive the data from stream and write to backup file */ remote_copy_file(file_backup_conn, file); - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied %d bytes", + file->path, file->write_size); PQfinish(file_backup_conn); } @@ -2099,27 +2099,24 @@ backup_files(void *arg) continue; } } - else - /* TODO: - * Check if file exists in previous backup - * If exists: - * if mtime > start_backup_time of parent backup, - * copy file to backup - * if mtime < start_backup_time - * calculate crc, compare crc to old file - * if crc is the same -> skip file - */ - if (!copy_file(arguments->from_root, - arguments->to_root, - file)) + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; } - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied %d bytes", + file->path, file->write_size); } else elog(LOG, "unexpected file type %d", buf.st_mode); diff --git a/src/data.c b/src/data.c index be07ce64..4605b275 100644 --- a/src/data.c +++ b/src/data.c @@ -229,8 +229,8 @@ backup_data_page(backup_files_args *arguments, BackupMode backup_mode) { BackupPageHeader header; - Page page = malloc(BLCKSZ); - Page compressed_page = NULL; + Page page = malloc(BLCKSZ); + Page compressed_page = NULL; XLogRecPtr page_lsn = 0; size_t write_buffer_size; char write_buffer[BLCKSZ+sizeof(header)]; @@ -385,7 +385,7 @@ backup_data_page(backup_files_args *arguments, /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", @@ -843,7 +843,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->read_size += read_len; } - file->write_size = file->read_size; + file->write_size = (int) file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index 20725f88..6cbd5f2e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -821,11 +821,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," + fprintf(out, "{\"path\":\"%s\", \"size\":\"%d\",\"mode\":\"%u\"," "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," "\"compress_alg\":\"%s\"", - path, (unsigned long) file->write_size, file->mode, - file->is_datafile?1:0, file->is_cfs?1:0, file->crc, + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg)); if (file->is_datafile) @@ -1032,7 +1032,7 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); - file->write_size = (size_t) write_size; + file->write_size = (int) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8face84d..15c29204 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,7 +88,7 @@ typedef struct pgFile size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - size_t write_size; /* size of the backed-up file. BYTES_INVALID means + int write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ pg_crc32 crc; /* CRC value of the file, regular file only */ @@ -160,7 +160,7 @@ typedef enum ShowFormat /* special values of pgBackup fields */ -#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) typedef struct pgBackupConfig diff --git a/src/restore.c b/src/restore.c index 0e42c4c4..3bf87d7c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -777,8 +777,8 @@ restore_files(void *arg) /* print size of restored file */ if (file->write_size != BYTES_INVALID) - elog(LOG, "Restored file %s : %lu bytes", - file->path, (unsigned long) file->write_size); + elog(LOG, "Restored file %s : %d bytes", + file->path, file->write_size); } /* Data files restoring is successful */ diff --git a/src/validate.c b/src/validate.c index cb693268..97a248ec 100644 --- a/src/validate.c +++ b/src/validate.c @@ -189,9 +189,8 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : %lu. Expected %lu", - file->path, (unsigned long) file->write_size, - (unsigned long) st.st_size); + elog(WARNING, "Invalid size of backup file \"%s\" : %d. Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); arguments->corrupted = true; break; } From 1da749aa5ea6a1e65b47ebed7e5e2f56bf725a76 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 13:55:30 +0300 Subject: [PATCH 43/99] Default compression level is 1, use NOT_DEFINED_COMPRESS for not defined compression algorithm --- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index c66179e1..668744b2 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -564,7 +564,7 @@ compress_init(void) if (backup_subcmd != SET_CONFIG) { if (compress_level != DEFAULT_COMPRESS_LEVEL - && compress_alg == NONE_COMPRESS) + && compress_alg == NOT_DEFINED_COMPRESS) elog(ERROR, "Cannot specify compress-level option without compress-alg option"); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 15c29204..b01094e3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -353,7 +353,7 @@ extern CompressAlg compress_alg; extern int compress_level; extern bool compress_shortcut; -#define DEFAULT_COMPRESS_LEVEL 6 +#define DEFAULT_COMPRESS_LEVEL 1 extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); From 03a3fb8a14558698e4646ea14acaf44f15697b06 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 15:14:44 +0300 Subject: [PATCH 44/99] PGPRO-1646: Use int64 for write_size --- src/backup.c | 6 +++--- src/data.c | 2 +- src/dir.c | 7 ++++--- src/pg_probackup.h | 3 ++- src/restore.c | 2 +- src/validate.c | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index c96d8f17..e58da8b8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -357,7 +357,7 @@ remote_copy_file(PGconn *conn, pgFile* file) elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); } - file->write_size = (int) file->read_size; + file->write_size = (int64) file->read_size; FIN_CRC32C(file->crc); fclose(out); @@ -438,7 +438,7 @@ remote_backup_files(void *arg) /* receive the data from stream and write to backup file */ remote_copy_file(file_backup_conn, file); - elog(VERBOSE, "File \"%s\". Copied %d bytes", + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", file->path, file->write_size); PQfinish(file_backup_conn); } @@ -2115,7 +2115,7 @@ backup_files(void *arg) continue; } - elog(VERBOSE, "File \"%s\". Copied %d bytes", + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", file->path, file->write_size); } else diff --git a/src/data.c b/src/data.c index 4605b275..4ab8a1c0 100644 --- a/src/data.c +++ b/src/data.c @@ -843,7 +843,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->read_size += read_len; } - file->write_size = (int) file->read_size; + file->write_size = (int64) file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index 6cbd5f2e..64058b62 100644 --- a/src/dir.c +++ b/src/dir.c @@ -821,8 +821,9 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"%d\",\"mode\":\"%u\"," - "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, @@ -1032,7 +1033,7 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); - file->write_size = (int) write_size; + file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b01094e3..b4ebfb98 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,9 +88,10 @@ typedef struct pgFile size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - int write_size; /* size of the backed-up file. BYTES_INVALID means + int64 write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ + /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ diff --git a/src/restore.c b/src/restore.c index 3bf87d7c..a35d50b2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -777,7 +777,7 @@ restore_files(void *arg) /* print size of restored file */ if (file->write_size != BYTES_INVALID) - elog(LOG, "Restored file %s : %d bytes", + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", file->path, file->write_size); } diff --git a/src/validate.c b/src/validate.c index 97a248ec..41467f25 100644 --- a/src/validate.c +++ b/src/validate.c @@ -189,7 +189,7 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : %d. Expected %lu", + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", file->path, file->write_size, (unsigned long) st.st_size); arguments->corrupted = true; break; From 1304c88ce74e0a95736f667b7bd107382526320e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 15:45:03 +0300 Subject: [PATCH 45/99] Minor style fixes --- src/backup.c | 12 ++++++------ src/configure.c | 3 +-- src/dir.c | 10 +++------- src/restore.c | 2 +- src/utils/logger.c | 4 ++-- src/utils/pgut.c | 5 ++--- 6 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index e58da8b8..e8efd4fb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -105,7 +105,7 @@ static void write_backup_file_list(parray *files, const char *root); static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); -static void StreamLog(void *arg); +static void *StreamLog(void *arg); static void get_remote_pgdata_filelist(parray *files); static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); @@ -599,8 +599,7 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; - pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, - &stream_thread_arg); + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } /* initialize backup list */ @@ -1051,8 +1050,7 @@ confirm_block_size(const char *name, int blcksz) "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); - PQclear(res);//bad pointer to endp - + PQclear(res); } /* @@ -2539,7 +2537,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* * Start the log streaming */ -static void +static void * StreamLog(void *arg) { XLogRecPtr startpos; @@ -2613,6 +2611,8 @@ StreamLog(void *arg) PQfinish(stream_arg->conn); stream_arg->conn = NULL; + + return NULL; } /* diff --git a/src/configure.c b/src/configure.c index 2fc27892..11c068a5 100644 --- a/src/configure.c +++ b/src/configure.c @@ -132,8 +132,7 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - //fprintf(out, "system-identifier = %li\n", config->system_identifier); - fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/dir.c b/src/dir.c index 64058b62..c29e4bbc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -100,12 +100,9 @@ static void dir_list_file_internal(parray *files, const char *root, int dir_create_dir(const char *dir, mode_t mode) { - char copy[MAXPGPATH]; - char *parent; + char parent[MAXPGPATH]; - strncpy(copy, dir, MAXPGPATH); - - parent = pstrdup(dir); + strncpy(parent, dir, MAXPGPATH); get_parent_directory(parent); /* Create parent first */ @@ -120,7 +117,6 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } - pfree(parent); return 0; } @@ -155,7 +151,7 @@ pgFileInit(const char *path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); - file->name = 0; + file->name = NULL; file->size = 0; file->mode = 0; diff --git a/src/restore.c b/src/restore.c index a35d50b2..dc339fd5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -92,7 +92,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, int dest_backup_index = 0; int base_full_backup_index = 0; int corrupted_backup_index = 0; - char *action = is_restore ? "Restore":"Validate"; + char *action = is_restore ? "Restore":"Validate"; if (is_restore) { diff --git a/src/utils/logger.c b/src/utils/logger.c index 8fd78739..7a642330 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -562,7 +562,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, PG_BINARY_R); + control_file = fopen(control, "r"); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -608,7 +608,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, PG_BINARY_W); + control_file = fopen(control, "w"); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 1a7aae0b..48f1496b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) //opts[i].val > 128 || + if (!isprint(opts[i].val)) continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1046,7 +1046,6 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; - len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1221,7 +1220,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "#\n\r\t\v");//removed space + i = j = strcspn(s, "#\n\r\t\v"); memcpy(dst, s, j); } From 0afc00439ce09c77fd312f3656de4bbd2b9a3d90 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 18:05:18 +0300 Subject: [PATCH 46/99] Use optind to parse subcommand name --- src/pg_probackup.c | 222 ++++++++++++++++++++++----------------------- src/pg_probackup.h | 27 +++--- src/utils/pgut.c | 9 +- 3 files changed, 128 insertions(+), 130 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 668744b2..9a61c5d0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -64,7 +64,7 @@ static char *target_inclusive; static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; -static char *target_action = NULL;; +static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; @@ -99,9 +99,10 @@ ShowFormat show_format = SHOW_PLAIN; /* current settings */ pgBackup current; -ProbackupSubcmd backup_subcmd; +ProbackupSubcmd backup_subcmd = NO_CMD; -bool help = false; +static bool help_opt = false; +static bool version_opt = false; static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); @@ -114,7 +115,8 @@ static void compress_init(void); static pgut_option options[] = { /* directory options */ - { 'b', 1, "help", &help, SOURCE_CMDLINE }, + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 'b', 'V', "version", &version_opt, SOURCE_CMDLINE }, { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, /* common options */ @@ -152,7 +154,7 @@ static pgut_option options[] = { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, /* TODO not implemented yet */ - { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, /* retention options */ { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, @@ -198,6 +200,9 @@ main(int argc, char *argv[]) /* Check if backup_path is directory. */ struct stat stat_buf; int rc; + int i, + len = 0, + allocated = 0; /* initialize configuration */ pgBackup_init(¤t); @@ -210,106 +215,94 @@ main(int argc, char *argv[]) */ main_tid = pthread_self(); - /* Parse subcommands and non-subcommand options */ - if (argc > 1) - { - if (strcmp(argv[1], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH; - else if (strcmp(argv[1], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET; - else if (strcmp(argv[1], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE; - else if (strcmp(argv[1], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE; - else if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE_SUBCMD; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG; - else if (strcmp(argv[1], "--help") == 0 - || strcmp(argv[1], "help") == 0 - || strcmp(argv[1], "-?") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) - { - if (argc == 2) - { -#ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); -#else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); -#endif - exit(0); - } - else if (strcmp(argv[2], "--help") == 0) - help_command(argv[1]); - else - elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); - } - else - elog(ERROR, "Unknown subcommand"); - } - /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ - if (backup_subcmd == BACKUP || - backup_subcmd == RESTORE || - backup_subcmd == VALIDATE || - backup_subcmd == DELETE_SUBCMD) + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) { - int i, - len = 0, - allocated = 0; + int arglen = strlen(argv[i]); - allocated = sizeof(char) * MAXPGPATH; - command = (char *) palloc(allocated); - - for (i = 0; i < argc; i++) + if (arglen + len > allocated) { - int arglen = strlen(argv[i]); - - if (arglen + len > allocated) - { - allocated *= 2; - command = repalloc(command, allocated); - } - - strncpy(command + len, argv[i], arglen); - len += arglen; - command[len++] = ' '; + allocated *= 2; + command = repalloc(command, allocated); } - command[len] = '\0'; + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; } + command[len] = '\0'; + /* Parse command line arguments */ pgut_getopt(argc, argv, options); - if (help) - help_command(argv[2]); + /* Process a command */ + if (optind < argc) + { + if (strcmp(argv[optind], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[optind], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[optind], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[optind], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[optind], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[optind], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[optind], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[optind], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[optind], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[optind], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[optind], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[optind], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[optind], "help") == 0) + { + if (argc - optind < 2) + help_pg_probackup(); + else + help_command(argv[optind + 1]); + } + else if (strcmp(argv[optind], "version") == 0) + version_opt = true; + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); + } + + if (help_opt) + { + if (backup_subcmd == NO_CMD || argc - optind < 1) + help_pg_probackup(); + else + help_command(argv[optind]); + } + else if (version_opt) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + return 0; + } + else if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); /* backup_path is required for all pg_probackup commands except help */ if (backup_path == NULL) @@ -343,7 +336,7 @@ main(int argc, char *argv[]) } /* Option --instance is required for all commands except init and show */ - if (backup_subcmd != INIT && backup_subcmd != SHOW && backup_subcmd != VALIDATE) + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) { if (instance_name == NULL) elog(ERROR, "required parameter not specified: --instance"); @@ -363,7 +356,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT && backup_subcmd != ADD_INSTANCE) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -375,7 +368,7 @@ main(int argc, char *argv[]) * Read options from env variables or from config file, * unless we're going to set them via set-config. */ - if (instance_name && backup_subcmd != SET_CONFIG) + if (instance_name && backup_subcmd != SET_CONFIG_CMD) { /* Read environment variables */ pgut_getopt_env(options); @@ -398,10 +391,10 @@ main(int argc, char *argv[]) /* Sanity check of --backup-id option */ if (backup_id_string_param != NULL) { - if (backup_subcmd != RESTORE - && backup_subcmd != VALIDATE - && backup_subcmd != DELETE_SUBCMD - && backup_subcmd != SHOW) + if (backup_subcmd != RESTORE_CMD + && backup_subcmd != VALIDATE_CMD + && backup_subcmd != DELETE_CMD + && backup_subcmd != SHOW_CMD) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -429,7 +422,7 @@ main(int argc, char *argv[]) pgdata_exclude_dir[i] = "pg_log"; } - if (backup_subcmd == VALIDATE || backup_subcmd == RESTORE) + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, @@ -445,17 +438,17 @@ main(int argc, char *argv[]) /* do actual operation */ switch (backup_subcmd) { - case ARCHIVE_PUSH: + case ARCHIVE_PUSH_CMD: return do_archive_push(wal_file_path, wal_file_name, file_overwrite); - case ARCHIVE_GET: + case ARCHIVE_GET_CMD: return do_archive_get(wal_file_path, wal_file_name); - case ADD_INSTANCE: + case ADD_INSTANCE_CMD: return do_add_instance(); - case DELETE_INSTANCE: + case DELETE_INSTANCE_CMD: return do_delete_instance(); - case INIT: + case INIT_CMD: return do_init(); - case BACKUP: + case BACKUP_CMD: { const char *backup_mode; time_t start_time; @@ -470,20 +463,20 @@ main(int argc, char *argv[]) return do_backup(start_time); } - case RESTORE: + case RESTORE_CMD: return do_restore_or_validate(current.backup_id, recovery_target_options, true); - case VALIDATE: + case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0) return do_validate_all(); else return do_restore_or_validate(current.backup_id, recovery_target_options, false); - case SHOW: + case SHOW_CMD: return do_show(current.backup_id); - case DELETE_SUBCMD: + case DELETE_CMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) @@ -494,10 +487,13 @@ main(int argc, char *argv[]) return do_retention_purge(); else return do_delete(current.backup_id); - case SHOW_CONFIG: + case SHOW_CONFIG_CMD: return do_configure(true); - case SET_CONFIG: + case SET_CONFIG_CMD: return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); } return 0; @@ -561,7 +557,7 @@ compress_init(void) if (compress_shortcut) compress_alg = ZLIB_COMPRESS; - if (backup_subcmd != SET_CONFIG) + if (backup_subcmd != SET_CONFIG_CMD) { if (compress_level != DEFAULT_COMPRESS_LEVEL && compress_alg == NOT_DEFINED_COMPRESS) @@ -574,7 +570,7 @@ compress_init(void) if (compress_level == 0) compress_alg = NOT_DEFINED_COMPRESS; - if (backup_subcmd == BACKUP || backup_subcmd == ARCHIVE_PUSH) + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ if (compress_alg == ZLIB_COMPRESS) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b4ebfb98..39e2db67 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -139,18 +139,19 @@ typedef enum BackupMode typedef enum ProbackupSubcmd { - INIT = 0, - ARCHIVE_PUSH, - ARCHIVE_GET, - ADD_INSTANCE, - DELETE_INSTANCE, - BACKUP, - RESTORE, - VALIDATE, - SHOW, - DELETE_SUBCMD, - SET_CONFIG, - SHOW_CONFIG + NO_CMD = 0, + INIT_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + SHOW_CMD, + DELETE_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD } ProbackupSubcmd; typedef enum ShowFormat @@ -367,7 +368,7 @@ extern ShowFormat show_format; /* current settings */ extern pgBackup current; -extern ProbackupSubcmd backup_subcmd; +extern ProbackupSubcmd backup_subcmd; /* in dir.c */ /* exclude directory list for $PGDATA file listing */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 48f1496b..f1b5babd 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -135,8 +135,10 @@ static const unit_conversion time_unit_conversion_table[] = static size_t option_length(const pgut_option opts[]) { - size_t len; + size_t len; + for (len = 0; opts && opts[len].type; len++) { } + return len; } @@ -156,7 +158,7 @@ option_has_arg(char type) static void option_copy(struct option dst[], const pgut_option opts[], size_t len) { - size_t i; + size_t i; for (i = 0; i < len; i++) { @@ -1047,13 +1049,12 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) size_t len; len = option_length(options); - longopts = pgut_newarray(struct option, len + 1); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); option_copy(longopts, options, len); optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - optind = 2; while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); From d3b1127ab3c4671391fdf50747175fc5370dd2c3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 18:14:47 +0300 Subject: [PATCH 47/99] Remove duplicate help command code --- src/pg_probackup.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9a61c5d0..a6fd22a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -271,10 +271,8 @@ main(int argc, char *argv[]) backup_subcmd = SHOW_CONFIG_CMD; else if (strcmp(argv[optind], "help") == 0) { - if (argc - optind < 2) - help_pg_probackup(); - else - help_command(argv[optind + 1]); + optind++; + help_opt = true; } else if (strcmp(argv[optind], "version") == 0) version_opt = true; @@ -284,7 +282,7 @@ main(int argc, char *argv[]) if (help_opt) { - if (backup_subcmd == NO_CMD || argc - optind < 1) + if (argc - optind < 1) help_pg_probackup(); else help_command(argv[optind]); From f336db64820ee751f67a34824c4fc68e508c70ad Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 12:01:18 +0300 Subject: [PATCH 48/99] Add include pg_getopt.h --- src/pg_probackup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a6fd22a3..1e4635e2 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,6 +17,7 @@ #include #include #include +#include "pg_getopt.h" const char *PROGRAM_VERSION = "2.0.17"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; From 0e14954483b46bbec15c5f65f4f0e94e2a5ffb18 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 12:32:06 +0300 Subject: [PATCH 49/99] Update windows project templates --- msvs/template.pg_probackup.vcxproj | 5 ++--- msvs/template.pg_probackup96.vcxproj | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 9b1d84dd..46a7b2c2 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -163,7 +163,6 @@ libc;%(IgnoreSpecificDefaultLibraries) - @@ -204,8 +203,8 @@ - - + + diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj index 185c6382..46e019ba 100644 --- a/msvs/template.pg_probackup96.vcxproj +++ b/msvs/template.pg_probackup96.vcxproj @@ -163,7 +163,6 @@ libc;%(IgnoreSpecificDefaultLibraries) - @@ -202,8 +201,8 @@ - - + + From b10fdf119de01ae07e8d4235e87931059751e2d4 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 16:22:25 +0300 Subject: [PATCH 50/99] Update command line parse --- src/pg_probackup.c | 73 +++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1e4635e2..75ae0314 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -241,44 +241,49 @@ main(int argc, char *argv[]) command[len] = '\0'; /* Parse command line arguments */ - pgut_getopt(argc, argv, options); - - /* Process a command */ - if (optind < argc) + optind = 1; + /* process command-line options */ + while (optind < argc) { - if (strcmp(argv[optind], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[optind], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[optind], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[optind], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[optind], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[optind], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[optind], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[optind], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[optind], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[optind], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[optind], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[optind], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[optind], "help") == 0) + pgut_getopt(argc, argv, options); + /* Process a command */ + if (optind < argc) { + if (strcmp(argv[optind], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[optind], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[optind], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[optind], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[optind], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[optind], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[optind], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[optind], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[optind], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[optind], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[optind], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[optind], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[optind], "help") == 0) + { + optind++; + help_opt = true; + } + else if (strcmp(argv[optind], "version") == 0) + version_opt = true; + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); optind++; - help_opt = true; } - else if (strcmp(argv[optind], "version") == 0) - version_opt = true; - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); } if (help_opt) From dac3958eecadc15d9529602c0f57c3e5a6f77100 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Jun 2018 19:42:19 +0300 Subject: [PATCH 51/99] Return command line parse algorithm --- src/help.c | 2 +- src/pg_probackup.c | 169 +++++++++++++++++++++++---------------------- 2 files changed, 87 insertions(+), 84 deletions(-) diff --git a/src/help.c b/src/help.c index 28500e75..1d147989 100644 --- a/src/help.c +++ b/src/help.c @@ -56,7 +56,7 @@ help_command(char *command) || strcmp(command, "-V") == 0) printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); else - printf(_("Unknown command. Try pg_probackup help\n")); + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); exit(0); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 75ae0314..e0cb5479 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -103,7 +103,6 @@ pgBackup current; ProbackupSubcmd backup_subcmd = NO_CMD; static bool help_opt = false; -static bool version_opt = false; static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); @@ -117,7 +116,6 @@ static pgut_option options[] = { /* directory options */ { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, - { 'b', 'V', "version", &version_opt, SOURCE_CMDLINE }, { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, /* common options */ @@ -196,14 +194,12 @@ static pgut_option options[] = int main(int argc, char *argv[]) { - char *command = NULL; + char *command = NULL, + *command_name; char path[MAXPGPATH]; /* Check if backup_path is directory. */ struct stat stat_buf; int rc; - int i, - len = 0, - allocated = 0; /* initialize configuration */ pgBackup_init(¤t); @@ -216,97 +212,104 @@ main(int argc, char *argv[]) */ main_tid = pthread_self(); + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ - allocated = sizeof(char) * MAXPGPATH; - command = (char *) palloc(allocated); - - for (i = 0; i < argc; i++) + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD) { - int arglen = strlen(argv[i]); + int i, + len = 0, + allocated = 0; - if (arglen + len > allocated) + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) { - allocated *= 2; - command = repalloc(command, allocated); - } + int arglen = strlen(argv[i]); - strncpy(command + len, argv[i], arglen); - len += arglen; - command[len++] = ' '; - } - - command[len] = '\0'; - - /* Parse command line arguments */ - optind = 1; - /* process command-line options */ - while (optind < argc) - { - pgut_getopt(argc, argv, options); - /* Process a command */ - if (optind < argc) - { - if (strcmp(argv[optind], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[optind], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[optind], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[optind], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[optind], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[optind], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[optind], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[optind], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[optind], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[optind], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[optind], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[optind], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[optind], "help") == 0) + if (arglen + len > allocated) { - optind++; - help_opt = true; + allocated *= 2; + command = repalloc(command, allocated); } - else if (strcmp(argv[optind], "version") == 0) - version_opt = true; - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); - optind++; + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; } + + command[len] = '\0'; } + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + if (help_opt) - { - if (argc - optind < 1) - help_pg_probackup(); - else - help_command(argv[optind]); - } - else if (version_opt) - { -#ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); -#else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); -#endif - return 0; - } - else if (backup_subcmd == NO_CMD) - elog(ERROR, "No subcommand specified"); + help_command(command_name); /* backup_path is required for all pg_probackup commands except help */ if (backup_path == NULL) From 6a95d7cf752f450f26888d0d22a3747c5b961dab Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 25 May 2018 18:56:35 +0300 Subject: [PATCH 52/99] PGPRO-427: A little optimization in process_block_change() --- src/backup.c | 40 ++++++++++++++++++++++++++++++---------- src/parsexlog.c | 1 + 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index e8efd4fb..06d536ab 100644 --- a/src/backup.c +++ b/src/backup.c @@ -130,6 +130,11 @@ static void check_system_identifiers(void); static void confirm_block_size(const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +/* + * A little optimization used in process_block_change(). + */ +static pgFile *last_used_file = NULL; + #define disconnect_and_exit(code) \ { \ @@ -500,6 +505,7 @@ do_backup_instance(void) } else current.tli = get_current_timeline(false); + /* * In incremental backup mode ensure that already-validated * backup on current timeline exists and get its filelist. @@ -509,6 +515,7 @@ do_backup_instance(void) current.backup_mode == BACKUP_MODE_DIFF_DELTA) { parray *backup_list; + /* get list of backups already taken */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); if (backup_list == NULL) @@ -628,6 +635,9 @@ do_backup_instance(void) */ if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) { + /* Just in case initialize it to NULL */ + last_used_file = NULL; + /* * Build the page map. Obtain information about changed pages * reading WAL segments present in archives up to the point @@ -2318,11 +2328,11 @@ datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno) void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) { - char *path; - char *rel_path; + char *path; + char *rel_path; BlockNumber blkno_inseg; int segno; - pgFile *file_item = NULL; + pgFile *file_item = NULL; int j; segno = blkno / RELSEG_SIZE; @@ -2332,14 +2342,24 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) path = pg_malloc(strlen(rel_path) + strlen(pgdata) + 2); sprintf(path, "%s/%s", pgdata, rel_path); - for (j = 0; j < parray_num(backup_files_list); j++) + /* + * Little optimization in case if we need the same file as in the previoius + * call. + */ + if (last_used_file && strcmp(last_used_file->path, path) == 0) + file_item = last_used_file; + else { - pgFile *p = (pgFile *) parray_get(backup_files_list, j); - - if (strcmp(p->path, path) == 0) + for (j = 0; j < parray_num(backup_files_list); j++) { - file_item = p; - break; + pgFile *p = (pgFile *) parray_get(backup_files_list, j); + + if (strcmp(p->path, path) == 0) + { + file_item = p; + last_used_file = p; + break; + } } } @@ -2443,7 +2463,7 @@ make_pagemap_from_ptrack(parray *files) } else { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); } diff --git a/src/parsexlog.c b/src/parsexlog.c index a0079b42..eacf9ec3 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -193,6 +193,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); + if (file->is_datafile && file->pagemap.bitmap == NULL) file->pagemap.bitmapsize = PageBitmapIsEmpty; } From 70257b371f23be190ec18a32dfe4da87adede724 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 28 May 2018 15:19:03 +0300 Subject: [PATCH 53/99] PGPRO-427: Remove unnecessary sorts, add binary search --- src/backup.c | 104 ++++++++++++++++----------------------------------- src/delete.c | 2 +- src/dir.c | 1 - 3 files changed, 34 insertions(+), 73 deletions(-) diff --git a/src/backup.c b/src/backup.c index 06d536ab..ab0a5916 100644 --- a/src/backup.c +++ b/src/backup.c @@ -130,12 +130,6 @@ static void check_system_identifiers(void); static void confirm_block_size(const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); -/* - * A little optimization used in process_block_change(). - */ -static pgFile *last_used_file = NULL; - - #define disconnect_and_exit(code) \ { \ if (conn != NULL) PQfinish(conn); \ @@ -618,6 +612,19 @@ do_backup_instance(void) else dir_list_file(backup_files_list, pgdata, true, true, false); + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); @@ -635,9 +642,6 @@ do_backup_instance(void) */ if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) { - /* Just in case initialize it to NULL */ - last_used_file = NULL; - /* * Build the page map. Obtain information about changed pages * reading WAL segments present in archives up to the point @@ -651,26 +655,16 @@ do_backup_instance(void) */ !current.from_replica, backup_files_list); } - - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - parray_qsort(backup_files_list, pgFileComparePath); + /* + * Build the page map from ptrack information. + */ make_pagemap_from_ptrack(backup_files_list); } /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - */ - parray_qsort(backup_files_list, pgFileComparePath); - - /* - * Make directories before backup - * and setup threads at the same time + * Make directories before backup and setup threads at the same time */ for (i = 0; i < parray_num(backup_files_list); i++) { @@ -771,12 +765,14 @@ do_backup_instance(void) for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + if (S_ISREG(file->mode)) calc_file_checksum(file); /* Remove file path root prefix*/ if (strstr(file->path, database_path) == file->path) { char *ptr = file->path; + file->path = pstrdup(GetRelativePath(ptr, database_path)); free(ptr); } @@ -2300,27 +2296,6 @@ write_backup_file_list(parray *files, const char *root) elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); } -/* - * A helper function to create the path of a relation file and segment. - * The returned path is palloc'd - */ -static char * -datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno) -{ - char *path; - char *segpath; - - path = relpathperm(rnode, forknum); - if (segno > 0) - { - segpath = psprintf("%s.%u", path, segno); - pfree(path); - return segpath; - } - else - return path; -} - /* * Find pgfile by given rnode in the backup_files_list * and add given blkno to its pagemap. @@ -2332,36 +2307,24 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) char *rel_path; BlockNumber blkno_inseg; int segno; - pgFile *file_item = NULL; - int j; + pgFile **file_item; + pgFile f; segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; - rel_path = datasegpath(rnode, forknum, segno); - path = pg_malloc(strlen(rel_path) + strlen(pgdata) + 2); - sprintf(path, "%s/%s", pgdata, rel_path); - - /* - * Little optimization in case if we need the same file as in the previoius - * call. - */ - if (last_used_file && strcmp(last_used_file->path, path) == 0) - file_item = last_used_file; + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); else - { - for (j = 0; j < parray_num(backup_files_list); j++) - { - pgFile *p = (pgFile *) parray_get(backup_files_list, j); + path = psprintf("%s/%s", pgdata, rel_path); - if (strcmp(p->path, path) == 0) - { - file_item = p; - last_used_file = p; - break; - } - } - } + pg_free(rel_path); + + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); /* * If we don't have any record of this file in the file map, it means @@ -2370,10 +2333,9 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) * backup would simply copy it as-is. */ if (file_item) - datapagemap_add(&file_item->pagemap, blkno_inseg); + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); pg_free(path); - pg_free(rel_path); } /* diff --git a/src/delete.c b/src/delete.c index f81fe70d..c933382e 100644 --- a/src/delete.c +++ b/src/delete.c @@ -281,7 +281,7 @@ pgBackupDeleteFiles(pgBackup *backup) /* print progress */ elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, - (unsigned long) parray_num(files), file->path); + (unsigned long) parray_num(files), file->path); if (remove(file->path)) { diff --git a/src/dir.c b/src/dir.c index c29e4bbc..616f6730 100644 --- a/src/dir.c +++ b/src/dir.c @@ -385,7 +385,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); - parray_qsort(files, pgFileComparePath); } /* From a3b9d1ebbb8cd20704730f265e6bb5d15409b782 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 29 May 2018 18:44:08 +0300 Subject: [PATCH 54/99] PGPRO-427: Add lock for datapagemap_add --- src/backup.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backup.c b/src/backup.c index ab0a5916..beca8bc8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -46,6 +46,9 @@ const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + /* * We need to wait end of WAL streaming before execute pg_stop_backup(). */ @@ -2333,8 +2336,17 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) * backup would simply copy it as-is. */ if (file_item) + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_mutex_lock(&backup_pagemap_mutex); + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } + pg_free(path); } From b97e9e8e29e9a1605f1f2487d7a7143fed1ad60f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 30 May 2018 13:43:52 +0300 Subject: [PATCH 55/99] PGPRO-427: Put in order thread arguments --- src/backup.c | 85 ++++++++++++++++++++++++---------------------- src/data.c | 4 +-- src/parsexlog.c | 18 +++++++++- src/pg_probackup.h | 18 +++++----- src/restore.c | 29 ++++++++-------- src/validate.c | 36 ++++++++++---------- 6 files changed, 108 insertions(+), 82 deletions(-) diff --git a/src/backup.c b/src/backup.c index beca8bc8..63a11bee 100644 --- a/src/backup.c +++ b/src/backup.c @@ -372,10 +372,10 @@ remote_copy_file(PGconn *conn, pgFile* file) static void * remote_backup_files(void *arg) { - int i; - backup_files_args *arguments = (backup_files_args *) arg; - int n_backup_files_list = parray_num(arguments->backup_files_list); - PGconn *file_backup_conn = NULL; + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; for (i = 0; i < n_backup_files_list; i++) { @@ -385,7 +385,7 @@ remote_backup_files(void *arg) pgFile *file; int row_length; - file = (pgFile *) parray_get(arguments->backup_files_list, i); + file = (pgFile *) parray_get(arguments->files_list, i); /* We have already copied all directories */ if (S_ISDIR(file->mode)) @@ -465,12 +465,11 @@ do_backup_instance(void) XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; /* arrays with meta info for multi threaded backup */ - pthread_t *backup_threads; - backup_files_args *backup_threads_args; + pthread_t *threads; + backup_files_arg *threads_args; bool backup_isok = true; pgBackup *prev_backup = NULL; - char prev_backup_filelist_path[MAXPGPATH]; parray *prev_backup_filelist = NULL; elog(LOG, "Database backup start"); @@ -512,6 +511,7 @@ do_backup_instance(void) current.backup_mode == BACKUP_MODE_DIFF_DELTA) { parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; /* get list of backups already taken */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -524,8 +524,8 @@ do_backup_instance(void) "Create new FULL backup before an incremental one."); parray_free(backup_list); - pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), - DATABASE_FILE_LIST); + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); @@ -701,20 +701,20 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileCompareSize); /* init thread args with own file lists */ - backup_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); - backup_threads_args = (backup_files_args *) palloc(sizeof(backup_files_args)*num_threads); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); for (i = 0; i < num_threads; i++) { - backup_files_args *arg = &(backup_threads_args[i]); + backup_files_arg *arg = &(threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; - arg->backup_files_list = backup_files_list; - arg->prev_backup_filelist = prev_backup_filelist; - arg->prev_backup_start_lsn = prev_backup_start_lsn; - arg->thread_backup_conn = NULL; - arg->thread_cancel_conn = NULL; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; /* By default there are some error */ arg->ret = 1; } @@ -723,20 +723,21 @@ do_backup_instance(void) elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { - backup_files_args *arg = &(backup_threads_args[i]); + backup_files_arg *arg = &(threads_args[i]); + elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) - pthread_create(&backup_threads[i], NULL, backup_files, arg); + pthread_create(&threads[i], NULL, backup_files, arg); else - pthread_create(&backup_threads[i], NULL, remote_backup_files, arg); + pthread_create(&threads[i], NULL, remote_backup_files, arg); } /* Wait threads */ for (i = 0; i < num_threads; i++) { - pthread_join(backup_threads[i], NULL); - if (backup_threads_args[i].ret == 1) + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) backup_isok = false; } if (backup_isok) @@ -2021,17 +2022,17 @@ backup_disconnect(bool fatal, void *userdata) static void * backup_files(void *arg) { - int i; - backup_files_args *arguments = (backup_files_args *) arg; - int n_backup_files_list = parray_num(arguments->backup_files_list); + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); /* backup a file */ for (i = 0; i < n_backup_files_list; i++) { int ret; struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -2077,11 +2078,15 @@ backup_files(void *arg) { int p; char *relative; - int n_prev_backup_files_list = parray_num(arguments->prev_backup_filelist); + int n_prev_files = parray_num(arguments->prev_filelist); + relative = GetRelativePath(file->path, arguments->from_root); - for (p = 0; p < n_prev_backup_files_list; p++) + for (p = 0; p < n_prev_files; p++) { - pgFile *prev_file = (pgFile *) parray_get(arguments->prev_backup_filelist, p); + pgFile *prev_file; + + prev_file = (pgFile *) parray_get(arguments->prev_filelist, p); + if (strcmp(relative, prev_file->path) == 0) { /* File exists in previous backup */ @@ -2098,7 +2103,7 @@ backup_files(void *arg) if (!backup_data_file(arguments, arguments->from_root, arguments->to_root, file, - arguments->prev_backup_start_lsn, + arguments->prev_start_lsn, current.backup_mode)) { file->write_size = BYTES_INVALID; @@ -2130,8 +2135,8 @@ backup_files(void *arg) } /* Close connection */ - if (arguments->thread_backup_conn) - pgut_disconnect(arguments->thread_backup_conn); + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); /* Data files transferring is successful */ arguments->ret = 0; @@ -2633,7 +2638,7 @@ get_last_ptrack_lsn(void) } char * -pg_ptrack_get_block(backup_files_args *arguments, +pg_ptrack_get_block(backup_files_arg *arguments, Oid dbOid, Oid tblsOid, Oid relOid, @@ -2658,17 +2663,17 @@ pg_ptrack_get_block(backup_files_args *arguments, sprintf(params[2], "%i", relOid); sprintf(params[3], "%u", blknum); - if (arguments->thread_backup_conn == NULL) + if (arguments->backup_conn == NULL) { - arguments->thread_backup_conn = pgut_connect(pgut_dbname); + arguments->backup_conn = pgut_connect(pgut_dbname); } - if (arguments->thread_cancel_conn == NULL) - arguments->thread_cancel_conn = PQgetCancel(arguments->thread_backup_conn); + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - res = pgut_execute_parallel(arguments->thread_backup_conn, - arguments->thread_cancel_conn, + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", 4, (const char **)params, true); diff --git a/src/data.c b/src/data.c index 4ab8a1c0..b874b1bd 100644 --- a/src/data.c +++ b/src/data.c @@ -221,7 +221,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * to the backup file. */ static void -backup_data_page(backup_files_args *arguments, +backup_data_page(backup_files_arg *arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, FILE *in, FILE *out, @@ -409,7 +409,7 @@ backup_data_page(backup_files_args *arguments, * backup with special header. */ bool -backup_data_file(backup_files_args* arguments, +backup_data_file(backup_files_arg* arguments, const char *from_root, const char *to_root, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode) diff --git a/src/parsexlog.c b/src/parsexlog.c index eacf9ec3..5d558ff4 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,7 +5,7 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ @@ -101,6 +101,19 @@ typedef struct XLogPageReadPrivate TimeLineID tli; } XLogPageReadPrivate; +/* An argument for a thread function */ +typedef struct +{ + parray *files; + bool corrupted; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, @@ -125,6 +138,9 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, XLogSegNo endSegNo, nextSegNo = 0; + pthread_t threads[num_threads]; + xlog_thread_arg thread_args[num_threads]; + elog(LOG, "Compiling pagemap"); if (!XRecOffIsValid(startpoint)) elog(ERROR, "Invalid startpoint value %X/%X", diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 39e2db67..887f6e5b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -276,18 +276,20 @@ typedef struct { const char *from_root; const char *to_root; - parray *backup_files_list; - parray *prev_backup_filelist; - XLogRecPtr prev_backup_start_lsn; - PGconn *thread_backup_conn; - PGcancel *thread_cancel_conn; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. */ int ret; -} backup_files_args; +} backup_files_arg; /* * return pointer that exceeds the length of prefix from character string. @@ -381,7 +383,7 @@ extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern char *pg_ptrack_get_block(backup_files_args *arguments, +extern char *pg_ptrack_get_block(backup_files_arg *arguments, Oid dbOid, Oid tblsOid, Oid relOid, BlockNumber blknum, size_t *result_size); @@ -485,7 +487,7 @@ extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); /* in data.c */ -extern bool backup_data_file(backup_files_args* arguments, +extern bool backup_data_file(backup_files_arg* arguments, const char *from_root, const char *to_root, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode); diff --git a/src/restore.c b/src/restore.c index dc339fd5..60bd32c2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -28,7 +28,7 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; -} restore_files_args; +} restore_files_arg; /* Tablespace mapping structures */ @@ -363,8 +363,8 @@ restore_backup(pgBackup *backup) parray *files; int i; /* arrays with meta info for multi threaded backup */ - pthread_t *restore_threads; - restore_files_args *restore_threads_args; + pthread_t *threads; + restore_files_arg *threads_args; bool restore_isok = true; if (backup->status != BACKUP_STATUS_OK) @@ -398,43 +398,44 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); files = dir_read_file_list(database_path, list_path); - restore_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); - restore_threads_args = (restore_files_args *) palloc(sizeof(restore_files_args)*num_threads); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); /* setup threads */ for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = &(restore_threads_args[i]); + restore_files_arg *arg = &(threads_args[i]); arg->files = files; arg->backup = backup; /* By default there are some error */ - arg->ret = 1; + threads_args[i].ret = 1; elog(LOG, "Start thread for num:%li", parray_num(files)); - pthread_create(&restore_threads[i], NULL, restore_files, arg); + pthread_create(&threads[i], NULL, restore_files, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { - pthread_join(restore_threads[i], NULL); - if (restore_threads_args[i].ret == 1) + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) restore_isok = false; } if (!restore_isok) elog(ERROR, "Data files restoring failed"); - pfree(restore_threads); - pfree(restore_threads_args); + pfree(threads); + pfree(threads_args); /* cleanup */ parray_walk(files, pgFileFree); @@ -710,7 +711,7 @@ static void * restore_files(void *arg) { int i; - restore_files_args *arguments = (restore_files_args *)arg; + restore_files_arg *arguments = (restore_files_arg *)arg; for (i = 0; i < parray_num(arguments->files); i++) { diff --git a/src/validate.c b/src/validate.c index 41467f25..61120d91 100644 --- a/src/validate.c +++ b/src/validate.c @@ -22,15 +22,15 @@ static bool corrupted_backup_found = false; typedef struct { - parray *files; - bool corrupted; + parray *files; + bool corrupted; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. */ int ret; -} validate_files_args; +} validate_files_arg; /* * Validate backup files. @@ -44,8 +44,8 @@ pgBackupValidate(pgBackup *backup) bool corrupted = false; bool validation_isok = true; /* arrays with meta info for multi threaded validate */ - pthread_t *validate_threads; - validate_files_args *validate_threads_args; + pthread_t *threads; + validate_files_arg *threads_args; int i; /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ @@ -83,29 +83,29 @@ pgBackupValidate(pgBackup *backup) } /* init thread args with own file lists */ - validate_threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - validate_threads_args = (validate_files_args *) - palloc(sizeof(validate_files_args) * num_threads); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = &(validate_threads_args[i]); + validate_files_arg *arg = &(threads_args[i]); arg->files = files; arg->corrupted = false; /* By default there are some error */ - arg->ret = 1; + threads_args[i].ret = 1; - pthread_create(&validate_threads[i], NULL, pgBackupValidateFiles, arg); + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = &(validate_threads_args[i]); + validate_files_arg *arg = &(threads_args[i]); - pthread_join(validate_threads[i], NULL); + pthread_join(threads[i], NULL); if (arg->corrupted) corrupted = true; if (arg->ret == 1) @@ -114,8 +114,8 @@ pgBackupValidate(pgBackup *backup) if (!validation_isok) elog(ERROR, "Data files validation failed"); - pfree(validate_threads); - pfree(validate_threads_args); + pfree(threads); + pfree(threads_args); /* cleanup */ parray_walk(files, pgFileFree); @@ -141,14 +141,14 @@ static void * pgBackupValidateFiles(void *arg) { int i; - validate_files_args *arguments = (validate_files_args *)arg; + validate_files_arg *arguments = (validate_files_arg *)arg; pg_crc32 crc; for (i = 0; i < parray_num(arguments->files); i++) { struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); - pgFile *file = (pgFile *) parray_get(arguments->files, i); if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -334,11 +334,13 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_CORRUPT) { int j; + corrupted_backup_found = true; current_backup_id = base36enc_dup(current_backup->start_time); for (j = i - 1; j >= 0; j--) { pgBackup *backup = (pgBackup *) parray_get(backups, j); + if (backup->backup_mode == BACKUP_MODE_FULL) break; if (backup->status != BACKUP_STATUS_OK) From 6e0f5bb68bb0e07ba7faf6d3353c358631524215 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 18 Jun 2018 11:47:29 +0300 Subject: [PATCH 56/99] PGPRO-427: Process blocks change in parallel --- src/backup.c | 2 +- src/parsexlog.c | 506 ++++++++++++++++++++++++++------------------- src/pg_probackup.h | 2 +- src/utils/logger.c | 18 +- src/utils/thread.c | 21 ++ src/utils/thread.h | 2 + tests/page.py | 60 +++++- 7 files changed, 382 insertions(+), 229 deletions(-) diff --git a/src/backup.c b/src/backup.c index 63a11bee..3f780e4e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2344,7 +2344,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) { /* We need critical section only we use more than one threads */ if (num_threads > 1) - pthread_mutex_lock(&backup_pagemap_mutex); + pthread_lock(&backup_pagemap_mutex); datapagemap_add(&(*file_item)->pagemap, blkno_inseg); diff --git a/src/parsexlog.c b/src/parsexlog.c index 5d558ff4..365dfb20 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -10,19 +10,17 @@ *------------------------------------------------------------------------- */ -#include "postgres_fe.h" - #include "pg_probackup.h" #include +#ifdef HAVE_LIBZ +#include +#endif #include "commands/dbcommands_xlog.h" #include "catalog/storage_xlog.h" #include "access/transam.h" - -#ifdef HAVE_LIBZ -#include -#endif +#include "utils/thread.h" /* * RmgrNames is an array of resource manager names, to make error messages @@ -85,27 +83,33 @@ typedef struct xl_xact_abort static void extractPageInfo(XLogReaderState *record); static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); -static int xlogreadfd = -1; -static XLogSegNo xlogreadsegno = -1; -static char xlogfpath[MAXPGPATH]; -static bool xlogexists = false; - -#ifdef HAVE_LIBZ -static gzFile gz_xlogread = NULL; -static char gz_xlogfpath[MAXPGPATH]; -#endif - typedef struct XLogPageReadPrivate { const char *archivedir; TimeLineID tli; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif } XLogPageReadPrivate; /* An argument for a thread function */ typedef struct { - parray *files; - bool corrupted; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; /* * Return value from the thread. @@ -118,57 +122,68 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* - * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the - * given timeline. Collect data blocks touched by the WAL records into a page map. - * - * If **prev_segno** is true then read all segments up to **endpoint** segment - * minus one. Else read all segments up to **endpoint** segment. + * extractPageMap() worker. */ -void -extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, - XLogRecPtr endpoint, bool prev_segno, parray *files) +static void * +doExtractPageMap(void *arg) { - size_t i; - XLogRecord *record; + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; char *errormsg; - XLogPageReadPrivate private; - XLogSegNo endSegNo, - nextSegNo = 0; - pthread_t threads[num_threads]; - xlog_thread_arg thread_args[num_threads]; - - elog(LOG, "Compiling pagemap"); - if (!XRecOffIsValid(startpoint)) - elog(ERROR, "Invalid startpoint value %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - - if (!XRecOffIsValid(endpoint)) - elog(ERROR, "Invalid endpoint value %X/%X", - (uint32) (endpoint >> 32), (uint32) (endpoint)); - - private.archivedir = archivedir; - private.tli = tli; - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, + &extract_arg->private_data); if (xlogreader == NULL) elog(ERROR, "out of memory"); - XLByteToSeg(endpoint, endSegNo); - if (prev_segno) - endSegNo--; - do { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; - errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + /* Try to switch to the next WAL segment */ + if (extract_arg->private_data.need_switch) + { + extract_arg->private_data.need_switch = false; + + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + extract_arg->private_data.xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (extract_arg->private_data.xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + XLogSegNoOffsetToRecPtr(extract_arg->private_data.xlogsegno, 0, + extract_arg->startpoint); + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; if (errormsg) elog(WARNING, "could not read WAL record at %X/%X: %s", @@ -183,38 +198,113 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, * start_lsn, we won't be able to build page map and PAGE backup will * be incorrect. Stop it and throw an error. */ - if (!xlogexists) - elog(ERROR, "WAL segment \"%s\" is absent", xlogfpath); - else if (xlogreadfd != -1) - elog(ERROR, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); + PrintXLogCorruptionMsg(&extract_arg->private_data, ERROR); } extractPageInfo(xlogreader); - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; XLByteToSeg(xlogreader->EndRecPtr, nextSegNo); - } while (nextSegNo <= endSegNo && xlogreader->EndRecPtr != endpoint); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, + XLogRecPtr endpoint, bool prev_seg, parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t threads[num_threads]; + xlog_thread_arg thread_args[num_threads]; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + XLByteToSeg(endpoint, endSegNo); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false); + thread_args[i].private_data.manual_switch = true; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + XLByteToSeg(startpoint, nextSegNoToRead); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint); + + threads_need++; } - /* Mark every datafile with empty pagemap as unchanged */ - for (i = 0; i < parray_num(files); i++) + /* Run threads */ + for (i = 0; i < threads_need; i++) { - pgFile *file = (pgFile *) parray_get(files, i); - - if (file->is_datafile && file->pagemap.bitmap == NULL) - file->pagemap.bitmapsize = PageBitmapIsEmpty; + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); } - elog(LOG, "Pagemap compiled"); + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + if (extract_isok) + elog(LOG, "Pagemap compiled"); + else + elog(ERROR, "Pagemap compiling failed"); } /* @@ -222,8 +312,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, */ static void validate_backup_wal_from_start_to_stop(pgBackup *backup, - char *backup_xlog_path, - TimeLineID tli) + char *backup_xlog_path, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; XLogRecord *record; @@ -232,15 +321,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, XLogPageReadPrivate private; bool got_endpoint = false; - private.archivedir = backup_xlog_path; - private.tli = tli; - - /* We will check it in the end */ - xlogfpath[0] = '\0'; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, true); while (true) { @@ -265,45 +346,27 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, if (!got_endpoint) { - if (xlogfpath[0] != 0) - { - /* XLOG reader couldn't read WAL segment. - * We throw a WARNING here to be able to update backup status below. - */ - if (!xlogexists) - { - elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); - } - else if (xlogreadfd != -1) - { - elog(WARNING, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); - } - } + PrintXLogCorruptionMsg(&private, WARNING); /* * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - backup->status = BACKUP_STATUS_CORRUPT; - pgBackupWriteBackupControlFile(backup); - elog(WARNING, "There are not enough WAL records to consistenly restore " - "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", - base36enc(backup->start_time), - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn), - (uint32) (backup->stop_lsn >> 32), - (uint32) (backup->stop_lsn)); + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); } /* clean */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } } /* @@ -386,15 +449,7 @@ validate_wal(pgBackup *backup, * up to the given recovery target. * In any case we cannot restore to the point before stop_lsn. */ - private.archivedir = archivedir; - - private.tli = tli; - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); - - /* We will check it in the end */ - xlogfpath[0] = '\0'; + xlogreader = InitXLogPageRead(&private, archivedir, tli, true); /* We can restore at least up to the backup end */ time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); @@ -456,21 +511,7 @@ validate_wal(pgBackup *backup, /* Some needed WAL records are absent */ else { - if (xlogfpath[0] != 0) - { - /* XLOG reader couldn't read WAL segment. - * We throw a WARNING here to be able to update backup status below. - */ - if (!xlogexists) - { - elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); - } - else if (xlogreadfd != -1) - { - elog(WARNING, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); - } - } + PrintXLogCorruptionMsg(&private, WARNING); elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, last_timestamp, last_xid); @@ -490,13 +531,8 @@ validate_wal(pgBackup *backup, } /* clean */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } } /* @@ -522,12 +558,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - private.archivedir = archivedir; - private.tli = tli; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, archivedir, tli, true); /* Read records from stop_lsn down to start_lsn */ do @@ -570,13 +601,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, res = false; cleanup: + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } return res; } @@ -598,23 +624,13 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "Invalid target_lsn value %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - private.archivedir = archivedir; - private.tli = target_tli; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, true); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } return res; } @@ -643,54 +659,48 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogPageReadPrivate *private_data; uint32 targetPageOff; + private_data = (XLogPageReadPrivate *) xlogreader->private_data; targetPageOff = targetPagePtr % XLogSegSize; /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (!XLByteInSeg(targetPagePtr, xlogreadsegno)) + if (!XLByteInSeg(targetPagePtr, private_data->xlogsegno)) { - if (xlogreadfd >= 0) + CleanupXLogPageRead(xlogreader); + if (private_data->manual_switch) { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; + private_data->need_switch = true; + return -1; } -#ifdef HAVE_LIBZ - else if (gz_xlogread != NULL) - { - gzclose(gz_xlogread); - gz_xlogread = NULL; - xlogexists = false; - } -#endif } - XLByteToSeg(targetPagePtr, xlogreadsegno); + XLByteToSeg(targetPagePtr, private_data->xlogsegno); - if (!xlogexists) + if (!private_data->xlogexists) { char xlogfname[MAXFNAMELEN]; - XLogFileName(xlogfname, private->tli, xlogreadsegno); - snprintf(xlogfpath, MAXPGPATH, "%s/%s", private->archivedir, - xlogfname); + XLogFileName(xlogfname, private_data->tli, private_data->xlogsegno); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); - if (fileExists(xlogfpath)) + if (fileExists(private_data->xlogpath)) { - elog(LOG, "Opening WAL segment \"%s\"", xlogfpath); + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); - xlogexists = true; - xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); - if (xlogreadfd < 0) + if (private_data->xlogfile < 0) { elog(WARNING, "Could not open WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } } @@ -698,17 +708,21 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* Try to open compressed WAL segment */ else { - snprintf(gz_xlogfpath, sizeof(gz_xlogfpath), "%s.gz", xlogfpath); - if (fileExists(gz_xlogfpath)) + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) { - elog(LOG, "Opening compressed WAL segment \"%s\"", gz_xlogfpath); + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); - xlogexists = true; - gz_xlogread = gzopen(gz_xlogfpath, "rb"); - if (gz_xlogread == NULL) + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) { elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", - gz_xlogfpath, strerror(errno)); + private_data->gz_xlogpath, strerror(errno)); return -1; } } @@ -716,55 +730,129 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #endif /* Exit without error if WAL segment doesn't exist */ - if (!xlogexists) + if (!private_data->xlogexists) return -1; } /* * At this point, we have the right segment open. */ - Assert(xlogexists); + Assert(private_data->xlogexists); /* Read the requested page */ - if (xlogreadfd != -1) + if (private_data->xlogfile != -1) { - if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) { elog(WARNING, "Could not seek in WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } - if (read(xlogreadfd, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Could not read from WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } } #ifdef HAVE_LIBZ else { - if (gzseek(gz_xlogread, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", - gz_xlogfpath, get_gz_error(gz_xlogread)); + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); return -1; } - if (gzread(gz_xlogread, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", - gz_xlogfpath, get_gz_error(gz_xlogread)); + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); return -1; } } #endif - *pageTLI = private->tli; + *pageTLI = private_data->tli; return XLOG_BLCKSZ; } +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlogfile = -1; + + if (allocate_reader) + { + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogpath != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + /* * Extract information about blocks modified in this record. */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 887f6e5b..18e4761a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -505,7 +505,7 @@ extern bool calc_file_checksum(pgFile *file); extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli, - XLogRecPtr endpoint, bool prev_segno, + XLogRecPtr endpoint, bool prev_seg, parray *backup_files_list); extern void validate_wal(pgBackup *backup, const char *archivedir, diff --git a/src/utils/logger.c b/src/utils/logger.c index 7a642330..4a85bf36 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -69,9 +69,6 @@ static bool exit_hook_registered = false; static bool loggin_in_progress = false; static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifdef WIN32 -static long mutex_initlock = 0; -#endif void init_logger(const char *root_path) @@ -173,20 +170,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) log_path[0] != '\0'; write_to_stderr = elevel >= LOG_LEVEL_CONSOLE && !file_only; - /* - * There is no need to lock if this is elog() from upper elog(). - */ -#ifdef WIN32 - if (log_file_mutex == NULL) - { - while (InterlockedExchange(&mutex_initlock, 1) == 1) - /* loop, another thread own the lock */ ; - if (log_file_mutex == NULL) - pthread_mutex_init(&log_file_mutex, NULL); - InterlockedExchange(&mutex_initlock, 0); - } -#endif - pthread_mutex_lock(&log_file_mutex); + pthread_lock(&log_file_mutex); loggin_in_progress = true; /* We need copy args only if we need write to error log file */ diff --git a/src/utils/thread.c b/src/utils/thread.c index a12a1c83..82c23764 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -22,6 +22,8 @@ typedef struct win32_pthread void *result; } win32_pthread; +static long mutex_initlock = 0; + static unsigned __stdcall win32_pthread_run(void *arg) { @@ -79,3 +81,22 @@ pthread_join(pthread_t th, void **thread_return) } #endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h index 1a50f1da..06460533 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -30,4 +30,6 @@ extern int pthread_join(pthread_t th, void **thread_return); extern pthread_t main_tid; +extern int pthread_lock(pthread_mutex_t *mp); + #endif /* PROBACKUP_THREAD_H */ diff --git a/tests/page.py b/tests/page.py index 9a7260c2..b0c870f6 100644 --- a/tests/page.py +++ b/tests/page.py @@ -4,7 +4,7 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess import time - +import ipdb module_name = 'page' @@ -526,3 +526,61 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.safe_psql("postgres", "create table test (id int)") + for x in range(0, 8): + node.safe_psql("postgres", + "insert into test select i from generate_series(1,100) s(i)") + self.switch_wal_segment(node) + count1 = node.safe_psql("postgres", "select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node(backup_dir, 'node', node, backup_type="page", + options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Check restored node + count2 = node.safe_psql("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) From bc016cfa61ec3f42060f9626785c7df1130feafb Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 19 Jun 2018 13:52:31 +0300 Subject: [PATCH 57/99] some change for removing visual studio build error: potentially uninitialized local pointer variable 'error_args' --- src/utils/logger.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 7a642330..fe69a9d8 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -190,14 +190,14 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) loggin_in_progress = true; /* We need copy args only if we need write to error log file */ - if (write_to_error_log) - va_copy(error_args, args); + //if (write_to_error_log) + // va_copy(error_args, args); /* * We need copy args only if we need write to stderr. But do not copy args * if we need to log only to stderr. */ - if (write_to_stderr && write_to_file) - va_copy(std_args, args); +// if (write_to_stderr && write_to_file) +// va_copy(std_args, args); if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", @@ -238,7 +238,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - + + va_copy(error_args, args); vfprintf(error_log_file, fmt, error_args); fputc('\n', error_log_file); fflush(error_log_file); @@ -253,7 +254,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_stderr) { write_elevel(stderr, elevel); - + va_copy(std_args, args); if (write_to_file) vfprintf(stderr, fmt, std_args); else From a2959609c8d692d8caca0050c7d85952f16161a8 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 19 Jun 2018 14:50:35 +0300 Subject: [PATCH 58/99] Some changes for windows --- src/utils/logger.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index fe69a9d8..22fc0508 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -177,6 +177,9 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) * There is no need to lock if this is elog() from upper elog(). */ #ifdef WIN32 + std_args = NULL; + error_args = NULL; + if (log_file_mutex == NULL) { while (InterlockedExchange(&mutex_initlock, 1) == 1) @@ -190,14 +193,14 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) loggin_in_progress = true; /* We need copy args only if we need write to error log file */ - //if (write_to_error_log) - // va_copy(error_args, args); + if (write_to_error_log) + va_copy(error_args, args); /* * We need copy args only if we need write to stderr. But do not copy args * if we need to log only to stderr. */ -// if (write_to_stderr && write_to_file) -// va_copy(std_args, args); + if (write_to_stderr && write_to_file) + va_copy(std_args, args); if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", @@ -239,7 +242,6 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - va_copy(error_args, args); vfprintf(error_log_file, fmt, error_args); fputc('\n', error_log_file); fflush(error_log_file); @@ -254,7 +256,6 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_stderr) { write_elevel(stderr, elevel); - va_copy(std_args, args); if (write_to_file) vfprintf(stderr, fmt, std_args); else From e2b2dee05e89d817f41eb10e9b3367313de52d80 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 22 Jun 2018 17:26:38 +0300 Subject: [PATCH 59/99] PGPRO-427: Add header size into WAL record start pointer --- src/parsexlog.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index 365dfb20..7c8420be 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -179,6 +179,9 @@ doExtractPageMap(void *arg) /* Adjust next record position */ XLogSegNoOffsetToRecPtr(extract_arg->private_data.xlogsegno, 0, extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint += SizeOfXLogLongPHD; + continue; } @@ -283,6 +286,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, if (nextSegNoToRead > endSegNo) break; XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; threads_need++; } From 6b539ad59a15367850cd65a85d42ee37a9b9fe36 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 28 Jun 2018 12:33:16 +0300 Subject: [PATCH 60/99] set 0 to PageBitmapIsAbsent --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 39e2db67..55a00636 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -112,7 +112,7 @@ typedef struct pgFile /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ -#define PageBitmapIsAbsent -1 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ +#define PageBitmapIsAbsent 0 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ /* Current state of backup */ typedef enum BackupStatus From 7f87c1cebebd0ab1a3a100d198a6940ee37f550c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 29 Jun 2018 14:01:08 +0300 Subject: [PATCH 61/99] Add pagemap_isabsent instead of PageBitmapIsAbsent define --- src/backup.c | 8 ++++---- src/data.c | 7 +++---- src/dir.c | 3 ++- src/pg_probackup.h | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index e8efd4fb..420bcb78 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2403,12 +2403,12 @@ make_pagemap_from_ptrack(parray *files) if (file->is_datafile) { - if (file->tblspcOid == tblspcOid_with_ptrack_init - && file->dbOid == dbOid_with_ptrack_init) + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) { /* ignore ptrack if ptrack_init exists */ elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; continue; } @@ -2461,7 +2461,7 @@ make_pagemap_from_ptrack(parray *files) * - target relation was deleted. */ elog(VERBOSE, "Ptrack is missing for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; } } } diff --git a/src/data.c b/src/data.c index 4ab8a1c0..fd09e5e7 100644 --- a/src/data.c +++ b/src/data.c @@ -430,7 +430,7 @@ backup_data_file(backup_files_args* arguments, if ((backup_mode == BACKUP_MODE_DIFF_PAGE || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev) + file->exists_in_prev && !file->pagemap_isabsent) { /* * There are no changed blocks since last backup. We want make @@ -494,9 +494,8 @@ backup_data_file(backup_files_args* arguments, * If page map is empty or file is not present in previous backup * backup all pages of the relation. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty - || file->pagemap.bitmapsize == PageBitmapIsAbsent - || !file->exists_in_prev) + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) { for (blknum = 0; blknum < nblocks; blknum++) { diff --git a/src/dir.c b/src/dir.c index c29e4bbc..0375a4de 100644 --- a/src/dir.c +++ b/src/dir.c @@ -161,7 +161,8 @@ pgFileInit(const char *path) file->is_datafile = false; file->linked = NULL; file->pagemap.bitmap = NULL; - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; file->tblspcOid = 0; file->dbOid = 0; file->relOid = 0; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 55a00636..75788cd7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -108,11 +108,12 @@ typedef struct pgFile CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ } pgFile; /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ -#define PageBitmapIsAbsent 0 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ /* Current state of backup */ typedef enum BackupStatus From 308022845329bb8e5318b390ad13877a510414d5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 4 Jul 2018 18:37:36 +0300 Subject: [PATCH 62/99] fix: add restore-target-name to validate --help. Submitted by Ludmila Mantrova. --- src/help.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/help.c b/src/help.c index 1d147989..b0225575 100644 --- a/src/help.c +++ b/src/help.c @@ -125,6 +125,7 @@ help_pg_probackup(void) printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); @@ -332,6 +333,8 @@ help_validate(void) printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From be8f3be6cd7b11e75c4a0e95a3703ba625b2b766 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 5 Jul 2018 13:19:18 +0300 Subject: [PATCH 63/99] PGPRO-1759: Fix messages and variable names, it should be data not date --- src/catalog.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index f19628c3..a0a8e5dc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -250,14 +250,14 @@ IsDir(const char *dirpath, const char *entry) parray * catalog_get_backup_list(time_t requested_backup_id) { - DIR *date_dir = NULL; - struct dirent *date_ent = NULL; + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; parray *backups = NULL; pgBackup *backup = NULL; /* open backup instance backups directory */ - date_dir = opendir(backup_instance_path); - if (date_dir == NULL) + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) { elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, strerror(errno)); @@ -266,21 +266,21 @@ catalog_get_backup_list(time_t requested_backup_id) /* scan the directory and list backups */ backups = parray_new(); - for (; (date_ent = readdir(date_dir)) != NULL; errno = 0) + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) { char backup_conf_path[MAXPGPATH]; - char date_path[MAXPGPATH]; + char data_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, date_ent->d_name) - || date_ent->d_name[0] == '.') + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(date_path, backup_instance_path, date_ent->d_name); + join_path_components(data_path, backup_instance_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ - snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); /* ignore corrupted backups */ @@ -298,8 +298,8 @@ catalog_get_backup_list(time_t requested_backup_id) if (errno && errno != ENOENT) { - elog(WARNING, "cannot read date directory \"%s\": %s", - date_ent->d_name, strerror(errno)); + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); goto err_proc; } } @@ -310,16 +310,16 @@ catalog_get_backup_list(time_t requested_backup_id) goto err_proc; } - closedir(date_dir); - date_dir = NULL; + closedir(data_dir); + data_dir = NULL; parray_qsort(backups, pgBackupCompareIdDesc); return backups; err_proc: - if (date_dir) - closedir(date_dir); + if (data_dir) + closedir(data_dir); if (backup) pgBackupFree(backup); if (backups) From 29639f71d60d5c799e23b8029599730a596b223a Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Thu, 5 Jul 2018 16:47:56 +0300 Subject: [PATCH 64/99] Fix --no-validate counting as recovery target --- src/restore.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index a5420a6c..8765a9e5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1079,7 +1079,6 @@ parseRecoveryTargetOptions(const char *target_time, if (restore_no_validate) { - recovery_target_specified++; rt->restore_no_validate = restore_no_validate; } From 1d5f54c5eedefceacd354dea4ea5b82720ffb819 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 6 Jul 2018 19:18:05 +0300 Subject: [PATCH 65/99] PGPRO-1376: Do not call dir_is_empty() twice, because readdir() may bring different ordering --- src/restore.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/restore.c b/src/restore.c index dc339fd5..a39b5127 100644 --- a/src/restore.c +++ b/src/restore.c @@ -581,14 +581,6 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) linked_path, dir_created, link_name); } - /* - * This check was done in check_tablespace_mapping(). But do - * it again. - */ - if (!dir_is_empty(linked_path)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - if (link_sep) elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", linked_path, From 1b70807d384d71c39558959b3355cf07864077d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Jul 2018 20:40:59 +0300 Subject: [PATCH 66/99] tests: fix archive tests --- tests/archive.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 92c0fbba..97fdca64 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -46,9 +46,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) # Recreate backup calagoue @@ -66,9 +65,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=["--recovery-target-action=promote"]) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) self.assertEqual( @@ -118,9 +116,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print(node.safe_psql( @@ -153,9 +150,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: @@ -185,10 +181,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) + if self.verbose: print('Fourth timeline') print(node.safe_psql( @@ -201,8 +197,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print('Fifth timeline') @@ -216,8 +212,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print('Sixth timeline') @@ -463,6 +459,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica, synchronous=True) + exit(1) self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.start() @@ -772,6 +769,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node(backup_dir, 'node', node) node.start() + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: + sleep(1) + self.assertEqual( result, node.safe_psql( @@ -803,7 +804,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL 10 for this test') else: - pg_receivexlog_path = node.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') pg_receivexlog = self.run_binary( [ @@ -843,6 +844,11 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node(backup_dir, 'node', node) node.start() + + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: + sleep(1) + self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), 'data after restore not equal to original data') From c81a61208c5ee1f8caf40d6a2b724d1242977167 Mon Sep 17 00:00:00 2001 From: Ilya Skvortsov Date: Mon, 9 Jul 2018 13:57:10 +0300 Subject: [PATCH 67/99] new no-validate test added --- tests/validate_test.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/validate_test.py b/tests/validate_test.py index 2cebcc33..1c9a3cc5 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1659,3 +1659,62 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message + + # Clean after yourself + self.del_test_dir(module_name, fname) From e947cac1428c8b8e0beb1210ab0b8366a7341604 Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Mon, 9 Jul 2018 20:35:59 +0700 Subject: [PATCH 68/99] Change INFO message in --no-validate mode --- src/restore.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 8765a9e5..228882aa 100644 --- a/src/restore.c +++ b/src/restore.c @@ -315,7 +315,12 @@ do_restore_or_validate(time_t target_backup_id, * produce corresponding error message */ if (dest_backup->status == BACKUP_STATUS_OK) - elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } else if (dest_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); else if (dest_backup->status == BACKUP_STATUS_ORPHAN) From e90714bcd44939c1a10eaf7d60d7a0e67852d84d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 10 Jul 2018 14:53:16 +0300 Subject: [PATCH 69/99] Push down path variable --- src/pg_probackup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e0cb5479..e6807dc0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -196,7 +196,6 @@ main(int argc, char *argv[]) { char *command = NULL, *command_name; - char path[MAXPGPATH]; /* Check if backup_path is directory. */ struct stat stat_buf; int rc; @@ -377,6 +376,8 @@ main(int argc, char *argv[]) */ if (instance_name && backup_subcmd != SET_CONFIG_CMD) { + char path[MAXPGPATH]; + /* Read environment variables */ pgut_getopt_env(options); From 963eb2e2c6bb795bf397c3ed97b1924d99287b63 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 11 Jul 2018 10:29:36 +0300 Subject: [PATCH 70/99] check for interrupt more frequently --- src/data.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/data.c b/src/data.c index fd09e5e7..476cbbdf 100644 --- a/src/data.c +++ b/src/data.c @@ -242,6 +242,10 @@ backup_data_page(backup_files_args *arguments, header.block = blknum; header.compressed_size = 0; + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + /* * Read the page and verify its header and checksum. * Under high write load it's possible that we've read partly From f98c91b3ba80fa2a149cd2fe26f3fd4d8c23be33 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 11 Jul 2018 10:50:38 +0300 Subject: [PATCH 71/99] tests: add slow_start() method to PostgresNode --- tests/archive.py | 51 ++++----------- tests/compression.py | 36 ++++++---- tests/delta.py | 10 +-- tests/expected/option_help.out | 3 + tests/false_positive.py | 4 +- tests/helpers/ptrack_helpers.py | 21 ++++++ tests/page.py | 55 ++++++---------- tests/ptrack.py | 43 +++--------- tests/restore_test.py | 112 ++++++-------------------------- tests/validate_test.py | 2 +- 10 files changed, 116 insertions(+), 221 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 97fdca64..6cd86e7d 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -45,10 +45,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() # Recreate backup calagoue self.init_pb(backup_dir) @@ -64,10 +61,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=["--recovery-target-action=promote"]) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -115,10 +109,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print(node.safe_psql( "postgres", @@ -149,10 +141,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() if self.verbose: print( @@ -180,10 +169,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() if self.verbose: print('Fourth timeline') @@ -196,10 +182,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print('Fifth timeline') print(node.safe_psql( @@ -211,10 +195,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print('Sixth timeline') print(node.safe_psql( @@ -459,7 +441,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica, synchronous=True) - exit(1) + self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.start() @@ -768,10 +750,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, @@ -843,11 +822,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() - - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), diff --git a/tests/compression.py b/tests/compression.py index eff42963..aa275382 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -83,7 +83,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -98,7 +99,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -113,7 +115,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -187,7 +190,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -202,7 +206,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -217,7 +222,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -294,7 +300,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -309,7 +316,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -324,7 +332,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -401,7 +410,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -416,7 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -431,7 +442,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() diff --git a/tests/delta.py b/tests/delta.py index 7cf21758..40450016 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -508,10 +508,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") @@ -946,11 +943,8 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 4080f4b8..a72aada8 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -25,6 +25,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--replica-timeout=timeout] pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-C] [--stream [-S slot-name]] [--backup-pg-log] @@ -61,10 +62,12 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup validate -B backup-dir [--instance=instance_name] [-i backup-id] [--progress] [--time=time|--xid=xid [--inclusive=boolean]] + [--recovery-target-name=target-name] [--timeline=timeline] pg_probackup show -B backup-dir [--instance=instance_name [-i backup-id]] + [--format=format] pg_probackup delete -B backup-dir --instance=instance_name [--wal] [-i backup-id | --expired] diff --git a/tests/false_positive.py b/tests/false_positive.py index 909d69fc..1884159b 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -191,7 +191,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, @@ -290,7 +290,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index af7fe766..4cdd8836 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -112,6 +112,25 @@ class ProbackupException(Exception): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +def slow_start(self): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() @@ -205,6 +224,8 @@ class ProbackupTest(object): os.makedirs(real_base_dir) node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) node.should_rm_dirs = True node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/tests/page.py b/tests/page.py index 9a7260c2..c3ca6ee9 100644 --- a/tests/page.py +++ b/tests/page.py @@ -3,7 +3,6 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess -import time module_name = 'page' @@ -33,8 +32,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir="{0}/{1}/node_restored".format(module_name, fname)) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -49,32 +47,27 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "create table t_heap tablespace somedata as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") + node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose'] - ) + options=['--log-level-file=verbose']) self.backup_node( - backup_dir, 'node', node, backup_type='page' - ) + backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -87,8 +80,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"] - ) + "--recovery-target-action=promote"]) # Physical comparison if self.paranoia: @@ -97,21 +89,17 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() - - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node_restored.slow_start() # Logical comparison result1 = node.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + result2 = node_restored.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + self.assertEqual(result1, result2) # Clean after yourself @@ -175,7 +163,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=full_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -188,7 +176,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=page_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -254,7 +242,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -271,7 +260,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -349,10 +339,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") diff --git a/tests/ptrack.py b/tests/ptrack.py index 4823acef..c2d6abff 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -586,10 +586,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -611,10 +608,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -691,11 +685,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) @@ -721,10 +711,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -1176,11 +1163,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1412,10 +1396,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + restored_node.slow_start() # COMPARE LOGICAL CONTENT result_new = restored_node.safe_psql( @@ -1450,11 +1431,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from t_heap") @@ -1553,11 +1530,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", diff --git a/tests/restore_test.py b/tests/restore_test.py index fce96911..862c8662 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -53,10 +53,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery_conf = os.path.join(node.data_dir, "recovery.conf") self.assertEqual(os.path.isfile(recovery_conf), True) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -104,10 +101,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -149,11 +143,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start(params=['-t', '10']) - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '10', '-c', '2', '--no-vacuum']) @@ -181,11 +171,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -234,11 +220,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -297,11 +279,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -366,11 +344,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -420,11 +394,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -479,11 +449,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -535,11 +501,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -602,11 +564,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -674,11 +632,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): options=["-j", "4", "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -769,10 +723,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test") self.assertEqual(result[0][0], 1) @@ -802,10 +754,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.execute("postgres", "SELECT id FROM test OFFSET 1") self.assertEqual(result[0][0], 2) @@ -881,10 +830,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) @@ -935,10 +881,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -987,11 +930,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1039,10 +978,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) @@ -1097,10 +1033,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1149,10 +1082,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-name=savepoint", "--recovery-target-action=promote"]) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result_new = node.safe_psql("postgres", "select * from t_heap") res = node.psql("postgres", "select * from t_heap_1") diff --git a/tests/validate_test.py b/tests/validate_test.py index afb2305f..29800bc9 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1320,7 +1320,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) node2.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) - node2.start() + node2.slow_start() timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] From 086cd4f7d418db1dfee1a67ec477e04ac22f0417 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 13 Jul 2018 19:30:48 +0300 Subject: [PATCH 72/99] Separated data.c => backup_data_page() into 2 functions: prepare_page() and compress_and_backup_page(). Additionally, fixed formatting in some places. (task pgpro-1726 in jira) --- src/data.c | 209 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 117 insertions(+), 92 deletions(-) diff --git a/src/data.c b/src/data.c index 476cbbdf..544fb98c 100644 --- a/src/data.c +++ b/src/data.c @@ -27,7 +27,7 @@ #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ -static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) { uLongf compressed_size = dst_size; int rc = compress2(dst, &compressed_size, src, src_size, compress_level); @@ -35,7 +35,7 @@ static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t } /* Implementation of zlib compression method */ -static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) { uLongf dest_len = dst_size; int rc = uncompress(dst, &dest_len, src, src_size); @@ -47,7 +47,7 @@ static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_ * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. */ -static size_t +static int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) { switch (alg) @@ -70,7 +70,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -static size_t +static int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) { switch (alg) @@ -101,6 +101,7 @@ typedef struct BackupPageHeader /* Special value for compressed_size field */ #define PageIsTruncated -2 +#define SkipCurrentPage -3 /* Verify page's header */ static bool @@ -134,8 +135,8 @@ static int read_page_from_file(pgFile *file, BlockNumber blknum, FILE *in, Page page, XLogRecPtr *page_lsn) { - off_t offset = blknum*BLCKSZ; - size_t read_len = 0; + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; /* read the block */ if (fseek(in, offset, SEEK_SET) != 0) @@ -216,32 +217,29 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* - * Backup the specified block from a file of a relation. - * Verify page header and checksum of the page and write it - * to the backup file. + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated */ -static void -backup_data_page(backup_files_args *arguments, +static int32 +prepare_page(backup_files_args *arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, - FILE *in, FILE *out, - pg_crc32 *crc, int *n_skipped, - BackupMode backup_mode) + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) { - BackupPageHeader header; - Page page = malloc(BLCKSZ); - Page compressed_page = NULL; - XLogRecPtr page_lsn = 0; - size_t write_buffer_size; - char write_buffer[BLCKSZ+sizeof(header)]; - - int try_again = 100; - bool page_is_valid = false; + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; - header.block = blknum; - header.compressed_size = 0; - /* check for interrupt */ if (interrupted) elog(ERROR, "Interrupted during backup"); @@ -262,7 +260,7 @@ backup_data_page(backup_files_args *arguments, if (result == 0) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; /* Page is not actually valid, but it is absent * and we're not going to reread it or validate */ page_is_valid = true; @@ -295,35 +293,38 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) { size_t page_size = 0; - - free(page); - page = NULL; - page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size); - if (page == NULL) + if (ptrack_page == NULL) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; } else if (page_size != BLCKSZ) { + free(ptrack_page); elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", file->path, absolute_blknum, BLCKSZ, page_size); } else { /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); if (is_checksum_enabled) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* get lsn from page, provided by pg_ptrack_get_block() */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && !parse_page(page, &page_lsn)) elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " "Possible risk of a memory corruption"); @@ -332,52 +333,70 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && page_lsn < prev_backup_start_lsn) { elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); (*n_skipped)++; - free(page); - return; + return SkipCurrentPage; } - if (header.compressed_size != PageIsTruncated) - { - file->read_size += BLCKSZ; + if (page_is_truncated) + return PageIsTruncated; - compressed_page = malloc(BLCKSZ); + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, compress_alg); + page, BLCKSZ, compress_alg); file->compress_alg = compress_alg; - + file->read_size += BLCKSZ; Assert (header.compressed_size <= BLCKSZ); - } - write_buffer_size = sizeof(header); - - /* - * The page was truncated. Write only header - * to know that we must truncate restored file - */ - if (header.compressed_size == PageIsTruncated) - { - memcpy(write_buffer, &header, sizeof(header)); - } - /* The page compression failed. Write it as is. */ - else if (header.compressed_size == -1) - { - header.compressed_size = BLCKSZ; - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), page, BLCKSZ); - write_buffer_size += header.compressed_size; - } - /* The page was successfully compressed */ - else if (header.compressed_size > 0) - { - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), compressed_page, header.compressed_size); - write_buffer_size += MAXALIGN(header.compressed_size); + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } } /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", @@ -389,7 +408,7 @@ backup_data_page(backup_files_args *arguments, /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", @@ -397,11 +416,6 @@ backup_data_page(backup_files_args *arguments, } file->write_size += write_buffer_size; - - if (page != NULL) - free(page); - if (compressed_page != NULL) - free(compressed_page); } /* @@ -418,13 +432,15 @@ backup_data_file(backup_files_args* arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode) { - char to_path[MAXPGPATH]; - FILE *in; - FILE *out; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int n_blocks_skipped = 0; - int n_blocks_read = 0; + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; /* * Skip unchanged file only if it exists in previous backup. @@ -503,9 +519,11 @@ backup_data_file(backup_files_args* arguments, { for (blknum = 0; blknum < nblocks; blknum++) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page); n_blocks_read++; } if (backup_mode == BACKUP_MODE_DIFF_DELTA) @@ -518,9 +536,11 @@ backup_data_file(backup_files_args* arguments, iter = datapagemap_iterate(&file->pagemap); while (datapagemap_next(iter, &blknum)) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page); n_blocks_read++; } @@ -635,7 +655,8 @@ restore_data_file(const char *from_root, } if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u",file->path, blknum); + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); if (header.compressed_size == PageIsTruncated) { @@ -646,7 +667,8 @@ restore_data_file(const char *from_root, if (ftruncate(fileno(out), header.block * BLCKSZ) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); - elog(VERBOSE, "truncate file %s to block %u", file->path, header.block); + elog(VERBOSE, "truncate file %s to block %u", + file->path, header.block); break; } @@ -664,10 +686,12 @@ restore_data_file(const char *from_root, uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, - header.compressed_size, file->compress_alg); + header.compressed_size, + file->compress_alg); if (uncompressed_size != BLCKSZ) - elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", uncompressed_size); + elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", + uncompressed_size); } /* @@ -714,7 +738,8 @@ restore_data_file(const char *from_root, if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); - elog(INFO, "Delta truncate file %s to block %u", file->path, file->n_blocks); + elog(INFO, "Delta truncate file %s to block %u", + file->path, file->n_blocks); } } From 2a1181ee92e42665a830277d3272e2451bf33c35 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Jul 2018 01:17:04 +0300 Subject: [PATCH 73/99] tests: minor fix --- tests/ptrack_cluster.py | 8 ++++---- tests/ptrack_truncate.py | 4 ++-- tests/ptrack_vacuum.py | 2 +- tests/ptrack_vacuum_bits_frozen.py | 4 ++-- tests/ptrack_vacuum_full.py | 2 +- tests/ptrack_vacuum_truncate.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index cf8702b8..784751ef 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -46,7 +46,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_btree') @@ -103,7 +103,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_gist') @@ -172,7 +172,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') @@ -242,7 +242,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py index 51e90503..928608c4 100644 --- a/tests/ptrack_truncate.py +++ b/tests/ptrack_truncate.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -100,7 +100,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index ad2f65bc..0409cae3 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index dc1e5fd8..f0cd3bbd 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -43,7 +43,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') @@ -111,7 +111,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'vacuum freeze t_heap') diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index a49d6075..ec12c9e2 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'vacuum full t_heap') diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 5fd85056..5c84c7e8 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id > 128;') node.safe_psql('postgres', 'vacuum t_heap') @@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id > 128;') From b67c0b54c4ec435b67136142d975412879b7290f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Jul 2018 02:45:17 +0300 Subject: [PATCH 74/99] add replica flag in slow_start() method --- tests/archive.py | 26 +++++++++++++------------- tests/helpers/ptrack_helpers.py | 28 +++++++++++++++++++--------- tests/replica.py | 26 +++++++++----------------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 6cd86e7d..8b8eb71a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -29,7 +29,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -89,7 +89,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FIRST TIMELINE node.safe_psql( @@ -247,7 +247,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.append_conf( 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( archive_script_path)) - node.start() + node.slow_start() try: self.backup_node( backup_dir, 'node', node, @@ -308,7 +308,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -368,7 +368,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -423,7 +423,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -444,7 +444,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -481,7 +481,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -513,7 +513,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -547,7 +547,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -573,7 +573,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -628,7 +628,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -655,7 +655,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA # self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 4cdd8836..78886455 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -112,7 +112,7 @@ class ProbackupException(Exception): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) -def slow_start(self): +def slow_start(self, replica=False): # wait for https://github.com/postgrespro/testgres/pull/50 # self.poll_query_until( @@ -121,14 +121,24 @@ def slow_start(self): # raise_operational_error=False) self.start() - while True: - try: - self.poll_query_until( - "postgres", - "SELECT not pg_is_in_recovery()") - break - except Exception as e: - continue + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + break + except Exception as e: + continue class ProbackupTest(object): diff --git a/tests/replica.py b/tests/replica.py index 5d6ecfb1..d95d0139 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -50,7 +50,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_replica(master, replica) # Check data correctness on replica - replica.start(["-t", "600"]) + replica.slow_start(replica=True) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -82,7 +82,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -113,7 +113,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -143,7 +143,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -166,7 +166,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start(["-t", "600"]) + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -200,7 +200,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -231,7 +231,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -260,7 +260,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -287,15 +287,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.start(["-t", "600"]) - - time.sleep(1) - self.assertEqual( - master.safe_psql( - "postgres", - "select exists(select * from pg_stat_replication)" - ).rstrip(), - 't') + replica.slow_start(replica=True) # Clean after yourself self.del_test_dir(module_name, fname) From f0864a2d4388ab75956b5965611db3decd4c6341 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Jul 2018 16:37:48 +0300 Subject: [PATCH 75/99] tests: replica slow_start() minor change --- tests/__init__.py | 4 ++++ tests/helpers/ptrack_helpers.py | 20 ++++++++++++-------- tests/replica.py | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d5f8b248..aeeabf2a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -60,6 +60,10 @@ def load_tests(loader, tests, pattern): # ptrack backup on replica should work correctly # archive: # immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored # 10vanilla_1.3ptrack + # 10vanilla+ # 9.6vanilla_1.3ptrack + diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 78886455..c430738c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -131,14 +131,18 @@ def slow_start(self, replica=False): except Exception as e: continue else: - while True: - try: - self.poll_query_until( - "postgres", - "SELECT pg_is_in_recovery()") - break - except Exception as e: - continue + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue class ProbackupTest(object): diff --git a/tests/replica.py b/tests/replica.py index d95d0139..d74c375c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -287,7 +287,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.slow_start(replica=True) + replica.start() # Clean after yourself self.del_test_dir(module_name, fname) From 4877f6b5d48842e6f9ead7f42431225d451fcdef Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 14:22:54 +0300 Subject: [PATCH 76/99] Issue #31: Use UTC time to make backup ID --- src/backup.c | 2 +- src/catalog.c | 6 +++--- src/delete.c | 2 +- src/parsexlog.c | 9 +++++---- src/pg_probackup.h | 2 +- src/restore.c | 5 +++-- src/show.c | 10 ++++++---- src/util.c | 26 +++++++++++++++----------- 8 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/backup.c b/src/backup.c index 420bcb78..666e8acc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -553,7 +553,7 @@ do_backup_instance(void) pg_ptrack_clear(); /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time); + time2iso(label, lengthof(label), current.start_time, true); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); diff --git a/src/catalog.c b/src/catalog.c index a0a8e5dc..9b3339dd 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -409,17 +409,17 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); fprintf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, false); fprintf(out, "end-time = '%s'\n", timestamp); } fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); fprintf(out, "recovery-time = '%s'\n", timestamp); } diff --git a/src/delete.c b/src/delete.c index f81fe70d..5b5c79c7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -257,7 +257,7 @@ pgBackupDeleteFiles(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) return 0; - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, true); elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); diff --git a/src/parsexlog.c b/src/parsexlog.c index a0079b42..a83acfe6 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -380,7 +380,8 @@ validate_wal(pgBackup *backup, xlogfpath[0] = '\0'; /* We can restore at least up to the backup end */ - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, + true); last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -430,7 +431,7 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time)); + timestamptz_to_time_t(last_time), true); /* There are all needed WAL records */ if (all_wal) @@ -459,8 +460,8 @@ validate_wal(pgBackup *backup, last_timestamp, last_xid); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time, + true); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 75788cd7..0a3be70b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -521,7 +521,7 @@ extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern void sanityChecks(void); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool to_local); extern const char *status2str(BackupStatus status); extern void remove_trailing_space(char *buf, int comment_mark); extern void remove_not_digit(char *buf, size_t len, const char *str); diff --git a/src/restore.c b/src/restore.c index a39b5127..1c295d23 100644 --- a/src/restore.c +++ b/src/restore.c @@ -122,7 +122,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup = (pgBackup *) parray_get(backups, i); /* Skip all backups which started after target backup */ - if (target_backup_id && current_backup->start_time > target_backup_id) + if (target_backup_id != INVALID_BACKUP_ID && + current_backup->start_time > target_backup_id) continue; /* @@ -381,7 +382,7 @@ restore_backup(pgBackup *backup) "XLOG_BLCKSZ(%d) is not compatible(%d expected)", backup->wal_block_size, XLOG_BLCKSZ); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, true); elog(LOG, "restoring database from backup %s", timestamp); /* diff --git a/src/show.c b/src/show.c index 01a558e0..d783ba06 100644 --- a/src/show.c +++ b/src/show.c @@ -324,7 +324,8 @@ show_instance_plain(parray *backup_list, bool show_name) char data_bytes_str[10] = "----"; if (backup->recovery_time != (time_t) 0) - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, + true); if (backup->end_time != (time_t) 0) snprintf(duration, lengthof(duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); @@ -452,12 +453,12 @@ show_instance_json(parray *backup_list) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, true); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, true); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -466,7 +467,8 @@ show_instance_json(parray *backup_list) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, + true); json_add_value(buf, "recovery-time", timestamp, json_level, true); } diff --git a/src/util.c b/src/util.c index a43239dc..7893f3da 100644 --- a/src/util.c +++ b/src/util.c @@ -194,31 +194,35 @@ get_data_checksum_version(bool safe) * Convert time_t value to ISO-8601 format string */ void -time2iso(char *buf, size_t len, time_t time) +time2iso(char *buf, size_t len, time_t time, bool to_local) { struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; - ptm = localtime(&time); - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); - - if (offset != 0) + if (to_local) { - buf += strlen(buf); - sprintf(buf, "%c%02d", + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", (offset >= 0) ? '+' : '-', abs((int) offset) / SECS_PER_HOUR); if (abs((int) offset) % SECS_PER_HOUR != 0) { - buf += strlen(buf); - sprintf(buf, ":%02d", + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } + else + strftime(buf, len, "%Y-%m-%d %H:%M:%S+00", ptm); } /* copied from timestamp.c */ From 188bb1119642872aea0606ee86ad3f7fdfb5899b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 15:02:50 +0300 Subject: [PATCH 77/99] Revert "Issue #31: Use UTC time to make backup ID" This reverts commit 4877f6b5d48842e6f9ead7f42431225d451fcdef. --- src/backup.c | 2 +- src/catalog.c | 6 +++--- src/delete.c | 2 +- src/parsexlog.c | 9 ++++----- src/pg_probackup.h | 2 +- src/restore.c | 5 ++--- src/show.c | 10 ++++------ src/util.c | 26 +++++++++++--------------- 8 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 666e8acc..420bcb78 100644 --- a/src/backup.c +++ b/src/backup.c @@ -553,7 +553,7 @@ do_backup_instance(void) pg_ptrack_clear(); /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time, true); + time2iso(label, lengthof(label), current.start_time); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); diff --git a/src/catalog.c b/src/catalog.c index 9b3339dd..a0a8e5dc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -409,17 +409,17 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time, false); + time2iso(timestamp, lengthof(timestamp), backup->start_time); fprintf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time, false); + time2iso(timestamp, lengthof(timestamp), backup->end_time); fprintf(out, "end-time = '%s'\n", timestamp); } fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); fprintf(out, "recovery-time = '%s'\n", timestamp); } diff --git a/src/delete.c b/src/delete.c index 5b5c79c7..f81fe70d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -257,7 +257,7 @@ pgBackupDeleteFiles(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) return 0; - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); diff --git a/src/parsexlog.c b/src/parsexlog.c index a83acfe6..a0079b42 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -380,8 +380,7 @@ validate_wal(pgBackup *backup, xlogfpath[0] = '\0'; /* We can restore at least up to the backup end */ - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, - true); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -431,7 +430,7 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time), true); + timestamptz_to_time_t(last_time)); /* There are all needed WAL records */ if (all_wal) @@ -460,8 +459,8 @@ validate_wal(pgBackup *backup, last_timestamp, last_xid); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), target_time, - true); + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0a3be70b..75788cd7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -521,7 +521,7 @@ extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern void sanityChecks(void); -extern void time2iso(char *buf, size_t len, time_t time, bool to_local); +extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); extern void remove_trailing_space(char *buf, int comment_mark); extern void remove_not_digit(char *buf, size_t len, const char *str); diff --git a/src/restore.c b/src/restore.c index 1c295d23..a39b5127 100644 --- a/src/restore.c +++ b/src/restore.c @@ -122,8 +122,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup = (pgBackup *) parray_get(backups, i); /* Skip all backups which started after target backup */ - if (target_backup_id != INVALID_BACKUP_ID && - current_backup->start_time > target_backup_id) + if (target_backup_id && current_backup->start_time > target_backup_id) continue; /* @@ -382,7 +381,7 @@ restore_backup(pgBackup *backup) "XLOG_BLCKSZ(%d) is not compatible(%d expected)", backup->wal_block_size, XLOG_BLCKSZ); - time2iso(timestamp, lengthof(timestamp), backup->start_time, true); + time2iso(timestamp, lengthof(timestamp), backup->start_time); elog(LOG, "restoring database from backup %s", timestamp); /* diff --git a/src/show.c b/src/show.c index d783ba06..01a558e0 100644 --- a/src/show.c +++ b/src/show.c @@ -324,8 +324,7 @@ show_instance_plain(parray *backup_list, bool show_name) char data_bytes_str[10] = "----"; if (backup->recovery_time != (time_t) 0) - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, - true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); if (backup->end_time != (time_t) 0) snprintf(duration, lengthof(duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); @@ -453,12 +452,12 @@ show_instance_json(parray *backup_list) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time, true); + time2iso(timestamp, lengthof(timestamp), backup->start_time); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time, true); + time2iso(timestamp, lengthof(timestamp), backup->end_time); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -467,8 +466,7 @@ show_instance_json(parray *backup_list) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, - true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); json_add_value(buf, "recovery-time", timestamp, json_level, true); } diff --git a/src/util.c b/src/util.c index 7893f3da..a43239dc 100644 --- a/src/util.c +++ b/src/util.c @@ -194,35 +194,31 @@ get_data_checksum_version(bool safe) * Convert time_t value to ISO-8601 format string */ void -time2iso(char *buf, size_t len, time_t time, bool to_local) +time2iso(char *buf, size_t len, time_t time) { struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; - if (to_local) + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); + + if (offset != 0) { - char *ptr = buf; - - ptm = localtime(&time); - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), "%c%02d", + buf += strlen(buf); + sprintf(buf, "%c%02d", (offset >= 0) ? '+' : '-', abs((int) offset) / SECS_PER_HOUR); if (abs((int) offset) % SECS_PER_HOUR != 0) { - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), ":%02d", + buf += strlen(buf); + sprintf(buf, ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } - else - strftime(buf, len, "%Y-%m-%d %H:%M:%S+00", ptm); } /* copied from timestamp.c */ From 059d3845cb080a7d3407d8591b546b9750578026 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 15:18:08 +0300 Subject: [PATCH 78/99] Issue #31: Times in backup.control is in UTC by default --- src/backup.c | 2 +- src/restore.c | 2 +- src/util.c | 26 ++++++++++++-------------- src/utils/pgut.c | 10 +++++++--- src/utils/pgut.h | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index 420bcb78..4b9bb1ff 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1836,7 +1836,7 @@ pg_stop_backup(pgBackup *backup) elog(ERROR, "result of txid_snapshot_xmax() is invalid: %s", PQgetvalue(res, 0, 0)); - if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time)) + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) elog(ERROR, "result of current_timestamp is invalid: %s", PQgetvalue(res, 0, 1)); diff --git a/src/restore.c b/src/restore.c index a39b5127..80bbb8bc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1036,7 +1036,7 @@ parseRecoveryTargetOptions(const char *target_time, rt->time_specified = true; rt->target_time_string = target_time; - if (parse_time(target_time, &dummy_time)) + if (parse_time(target_time, &dummy_time, false)) rt->recovery_target_time = dummy_time; else elog(ERROR, "Invalid value of --time option %s", target_time); diff --git a/src/util.c b/src/util.c index a43239dc..051f4d54 100644 --- a/src/util.c +++ b/src/util.c @@ -191,7 +191,7 @@ get_data_checksum_version(bool safe) /* - * Convert time_t value to ISO-8601 format string + * Convert time_t value to ISO-8601 format string. Always set timezone offset. */ void time2iso(char *buf, size_t len, time_t time) @@ -199,25 +199,23 @@ time2iso(char *buf, size_t len, time_t time) struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; + char *ptr = buf; ptm = localtime(&time); offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - if (offset != 0) + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) { - buf += strlen(buf); - sprintf(buf, "%c%02d", - (offset >= 0) ? '+' : '-', - abs((int) offset) / SECS_PER_HOUR); - - if (abs((int) offset) % SECS_PER_HOUR != 0) - { - buf += strlen(buf); - sprintf(buf, ":%02d", - abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); - } + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index f1b5babd..330f2fa5 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -256,7 +256,8 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) message = "a valid string. But provided: "; break; case 't': - if (parse_time(optarg, opt->var)) + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) return; message = "a time"; break; @@ -746,9 +747,12 @@ parse_uint64(const char *value, uint64 *result, int flags) /* * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. */ bool -parse_time(const char *value, time_t *result) +parse_time(const char *value, time_t *result, bool utc_default) { size_t len; int fields_num, @@ -870,7 +874,7 @@ parse_time(const char *value, time_t *result) *result = mktime(&tm); /* adjust time zone */ - if (tz_set) + if (tz_set || utc_default) { time_t ltime = time(NULL); struct tm *ptm = gmtime(<ime); diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a27cea83..662f07c1 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -204,7 +204,7 @@ extern bool parse_int32(const char *value, int32 *result, int flags); extern bool parse_uint32(const char *value, uint32 *result, int flags); extern bool parse_int64(const char *value, int64 *result, int flags); extern bool parse_uint64(const char *value, uint64 *result, int flags); -extern bool parse_time(const char *value, time_t *result); +extern bool parse_time(const char *value, time_t *result, bool utc_default); extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg); From 2650fdb78df61ef7b5e62ae66098eebf938b6724 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 17 Jul 2018 17:17:29 +0300 Subject: [PATCH 79/99] Added the gen_probackup_project.pl script for project generation --- gen_probackup_project.pl | 190 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 gen_probackup_project.pl diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 00000000..3ea79e96 --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} From f0abaf94389b360a58706087769d4772377a5af5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Jul 2018 09:10:38 +0300 Subject: [PATCH 80/99] add --no-validate description to help --- src/help.c | 3 ++- tests/expected/option_help.out | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index 773f6c9d..549a2e2b 100644 --- a/src/help.c +++ b/src/help.c @@ -121,6 +121,7 @@ help_pg_probackup(void) printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); @@ -285,7 +286,6 @@ help_restore(void) printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); - printf(_(" --no-validate disable backup validation during recovery\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --recovery-target-action=pause|promote|shutdown\n")); @@ -294,6 +294,7 @@ help_restore(void) printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index a72aada8..f9c59e0e 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -58,6 +58,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] + [--no-validate] pg_probackup validate -B backup-dir [--instance=instance_name] [-i backup-id] [--progress] From b122966a3ac76463896c0a26b97148f27e9adcc4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Jul 2018 09:41:45 +0300 Subject: [PATCH 81/99] PGPRO-427: additional test added --- tests/page.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/tests/page.py b/tests/page.py index b0c870f6..a189620b 100644 --- a/tests/page.py +++ b/tests/page.py @@ -4,7 +4,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess import time -import ipdb module_name = 'page' @@ -558,14 +557,72 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Fill instance with data and make several WAL segments ... node.safe_psql("postgres", "create table test (id int)") for x in range(0, 8): - node.safe_psql("postgres", + node.safe_psql( + "postgres", "insert into test select i from generate_series(1,100) s(i)") self.switch_wal_segment(node) count1 = node.safe_psql("postgres", "select count(*) from test") # ... and do page backup with parallel pagemap - self.backup_node(backup_dir, 'node', node, backup_type="page", - options=["-j", "4"]) + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Check restored node + count2 = node.safe_psql("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) show_backup = self.show_pb(backup_dir, 'node')[1] self.assertEqual(show_backup['status'], "OK") From 64262950c3c8dd4fbbd05a1dc2916e27220dc05f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Jul 2018 09:45:33 +0300 Subject: [PATCH 82/99] PGPRO-427: small fix to tests --- tests/page.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/page.py b/tests/page.py index a189620b..fe1def54 100644 --- a/tests/page.py +++ b/tests/page.py @@ -633,11 +633,6 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node', node) node.start() - # Check restored node - count2 = node.safe_psql("postgres", "select count(*) from test") - - self.assertEqual(count1, count2) - # Clean after yourself node.cleanup() self.del_test_dir(module_name, fname) From 6577c9b8e8adcfbcaea392fee3e4d5ee046ac4aa Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 18 Jul 2018 14:41:03 +0300 Subject: [PATCH 83/99] PGPRO-427: Use xlogreader API to locate right start LSN --- src/parsexlog.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7c8420be..8bc7175b 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -105,6 +105,7 @@ typedef struct XLogPageReadPrivate /* An argument for a thread function */ typedef struct { + int thread_num; XLogPageReadPrivate private_data; XLogRecPtr startpoint; @@ -148,6 +149,14 @@ doExtractPageMap(void *arg) if (xlogreader == NULL) elog(ERROR, "out of memory"); + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + do { XLogRecord *record; @@ -180,7 +189,13 @@ doExtractPageMap(void *arg) XLogSegNoOffsetToRecPtr(extract_arg->private_data.xlogsegno, 0, extract_arg->startpoint); /* Skip over the page header */ - extract_arg->startpoint += SizeOfXLogLongPHD; + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); continue; } @@ -266,6 +281,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, for (i = 0; i < num_threads; i++) { InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false); + thread_args[i].thread_num = i; thread_args[i].private_data.manual_switch = true; thread_args[i].startpoint = startpoint; From 56ce573832a9f77717b42901e7567b4d83ed980b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 18 Jul 2018 19:30:42 +0300 Subject: [PATCH 84/99] PGPRO-427: Count pagemap compiling time --- src/parsexlog.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 8bc7175b..71e6ddc5 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -12,6 +12,7 @@ #include "pg_probackup.h" +#include #include #ifdef HAVE_LIBZ #include @@ -256,6 +257,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, bool extract_isok = true; pthread_t threads[num_threads]; xlog_thread_arg thread_args[num_threads]; + time_t start_time, + end_time; elog(LOG, "Compiling pagemap"); if (!XRecOffIsValid(startpoint)) @@ -271,6 +274,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, endSegNo--; nextSegNoToRead = 0; + time(&start_time); /* * Initialize thread args. @@ -322,8 +326,11 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, if (thread_args[i].ret == 1) extract_isok = false; } + + time(&end_time); if (extract_isok) - elog(LOG, "Pagemap compiled"); + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); else elog(ERROR, "Pagemap compiling failed"); } From 4c3a86f3b0e104ee55424a0dec4ee87aeb8117fb Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 18 Jul 2018 19:44:11 +0300 Subject: [PATCH 85/99] Fix bug: count CRC for data not for pointer --- src/backup.c | 14 +++++++------- src/data.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4b9bb1ff..010b5aa3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -327,7 +327,7 @@ remote_copy_file(PGconn *conn, pgFile* file) { write_buffer_size = Min(row_length, sizeof(buf)); memcpy(buf, copybuf, write_buffer_size); - COMP_CRC32C(file->crc, &buf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); /* TODO calc checksum*/ if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) @@ -724,7 +724,7 @@ do_backup_instance(void) else pthread_create(&backup_threads[i], NULL, remote_backup_files, arg); } - + /* Wait threads */ for (i = 0; i < num_threads; i++) { @@ -844,7 +844,7 @@ do_backup(time_t start_time) elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " "pg_probackup have no way to detect data block corruption without them. " "Reinitialize PGDATA with option '--data-checksums'."); - + StrNCpy(current.server_version, server_version_str, sizeof(current.server_version)); current.stream = stream_wal; @@ -1020,7 +1020,7 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - + if (system_id_conn != system_identifier) elog(ERROR, "Backup data directory was initialized for system id %ld, but connected instance system id is %ld", system_identifier, system_id_conn); @@ -1043,13 +1043,13 @@ confirm_block_size(const char *name, int blcksz) res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); - + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); - + PQclear(res); } @@ -2024,7 +2024,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (!pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ diff --git a/src/data.c b/src/data.c index 544fb98c..72d8a1ba 100644 --- a/src/data.c +++ b/src/data.c @@ -403,7 +403,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, blknum, header.compressed_size, write_buffer_size); */ /* Update CRC */ - COMP_CRC32C(*crc, &write_buffer, write_buffer_size); + COMP_CRC32C(*crc, write_buffer, write_buffer_size); /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) From 87fb4fae93953d3bd81824c77c366051d701cdff Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 24 Jul 2018 15:48:44 +0300 Subject: [PATCH 86/99] Fix bug: get_control_value() should use signed long int; refactor restore_data_file(), so the loop will break in first truncated block --- src/data.c | 75 +++++++++++++++++++++++++++++++++------------- src/dir.c | 62 +++++++++++++++++++++----------------- src/pg_probackup.h | 1 + 3 files changed, 90 insertions(+), 48 deletions(-) diff --git a/src/data.c b/src/data.c index 72d8a1ba..5cff8784 100644 --- a/src/data.c +++ b/src/data.c @@ -513,6 +513,8 @@ backup_data_file(backup_files_args* arguments, * Read each page, verify checksum and write it to backup. * If page map is empty or file is not present in previous backup * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. */ if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev) @@ -525,11 +527,17 @@ backup_data_file(backup_files_args* arguments, compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page); n_blocks_read++; + if (page_state == PageIsTruncated) + break; } if (backup_mode == BACKUP_MODE_DIFF_DELTA) file->n_blocks = n_blocks_read; } - /* If page map is not empty we scan only changed blocks, */ + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ else { datapagemap_iterator_t *iter; @@ -542,6 +550,8 @@ backup_data_file(backup_files_args* arguments, compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page); n_blocks_read++; + if (page_state == PageIsTruncated) + break; } pg_free(file->pagemap.bitmap); @@ -596,8 +606,9 @@ restore_data_file(const char *from_root, FILE *in = NULL; FILE *out = NULL; BackupPageHeader header; - BlockNumber blknum; - size_t file_size; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) @@ -628,7 +639,7 @@ restore_data_file(const char *from_root, to_path, strerror(errno_tmp)); } - for (blknum = 0; ; blknum++) + while (true) { size_t read_len; DataPage compressed_page; /* used as read buffer */ @@ -638,6 +649,21 @@ restore_data_file(const char *from_root, if (file->write_size == BYTES_INVALID) break; + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); if (read_len != sizeof(header)) @@ -658,17 +684,16 @@ restore_data_file(const char *from_root, elog(ERROR, "backup is broken at file->path %s block %u", file->path, blknum); + blknum = header.block; + if (header.compressed_size == PageIsTruncated) { /* * Backup contains information that this block was truncated. - * Truncate file to this length. + * We need to truncate file to this length. */ - if (ftruncate(fileno(out), header.block * BLCKSZ) != 0) - elog(ERROR, "cannot truncate \"%s\": %s", - file->path, strerror(errno)); - elog(VERBOSE, "truncate file %s to block %u", - file->path, header.block); + truncate_from = blknum; + need_truncate = true; break; } @@ -697,7 +722,6 @@ restore_data_file(const char *from_root, /* * Seek and write the restored page. */ - blknum = header.block; if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); @@ -724,25 +748,34 @@ restore_data_file(const char *from_root, * So when restoring file from DELTA backup we, knowning it`s size at * a time of a backup, can truncate file to this size. */ - - if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA) + if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA && + file->n_blocks != BLOCKNUM_INVALID && !need_truncate) { + size_t file_size = 0; + /* get file current size */ fseek(out, 0, SEEK_END); file_size = ftell(out); + if (file_size > file->n_blocks * BLCKSZ) { - /* - * Truncate file to this length. - */ - if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0) - elog(ERROR, "cannot truncate \"%s\": %s", - file->path, strerror(errno)); - elog(INFO, "Delta truncate file %s to block %u", - file->path, file->n_blocks); + truncate_from = file->n_blocks; + need_truncate = true; } } + if (need_truncate) + { + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), truncate_from * BLCKSZ) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + /* update file permission */ if (chmod(to_path, file->mode) == -1) { diff --git a/src/dir.c b/src/dir.c index 0375a4de..d94f7095 100644 --- a/src/dir.c +++ b/src/dir.c @@ -186,7 +186,8 @@ pgFileInit(const char *path) file->is_cfs = false; file->exists_in_prev = false; /* can change only in Incremental backup. */ - file->n_blocks = -1; /* can change only in DELTA backup. Number of blocks readed during backup */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; return file; } @@ -836,7 +837,7 @@ print_file_list(FILE *out, const parray *files, const char *root) #endif fprintf(out, ",\"linked\":\"%s\"", file->linked); - if (file->n_blocks != -1) + if (file->n_blocks != BLOCKNUM_INVALID) fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); fprintf(out, "}\n"); @@ -858,23 +859,25 @@ print_file_list(FILE *out, const parray *files, const char *root) * {"name1":"value1", "name2":"value2"} * * The value will be returned to "value_str" as string if it is not NULL. If it - * is NULL the value will be returned to "value_ulong" as unsigned long. + * is NULL the value will be returned to "value_uint64" as int64. + * + * Returns true if the value was found in the line. */ -static void +static bool get_control_value(const char *str, const char *name, - char *value_str, uint64 *value_uint64, bool is_mandatory) + char *value_str, int64 *value_int64, bool is_mandatory) { int state = CONTROL_WAIT_NAME; char *name_ptr = (char *) name; char *buf = (char *) str; - char buf_uint64[32], /* Buffer for "value_uint64" */ - *buf_uint64_ptr = buf_uint64; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; /* Set default values */ if (value_str) *value_str = '\0'; - else if (value_uint64) - *value_uint64 = 0; + else if (value_int64) + *value_int64 = 0; while (*buf) { @@ -909,7 +912,7 @@ get_control_value(const char *str, const char *name, if (*buf == '"') { state = CONTROL_INVALUE; - buf_uint64_ptr = buf_uint64; + buf_int64_ptr = buf_int64; } else if (IsAlpha(*buf)) goto bad_format; @@ -922,19 +925,19 @@ get_control_value(const char *str, const char *name, { *value_str = '\0'; } - else if (value_uint64) + else if (value_int64) { /* Length of buf_uint64 should not be greater than 31 */ - if (buf_uint64_ptr - buf_uint64 >= 32) + if (buf_int64_ptr - buf_int64 >= 32) elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", name, str, DATABASE_FILE_LIST); - *buf_uint64_ptr = '\0'; - if (!parse_uint64(buf_uint64, value_uint64, 0)) + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) goto bad_format; } - return; + return true; } else { @@ -945,8 +948,8 @@ get_control_value(const char *str, const char *name, } else { - *buf_uint64_ptr = *buf; - buf_uint64_ptr++; + *buf_int64_ptr = *buf; + buf_int64_ptr++; } } break; @@ -970,11 +973,12 @@ get_control_value(const char *str, const char *name, if (is_mandatory) elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", name, str, DATABASE_FILE_LIST); - return; + return false; bad_format: elog(ERROR, "%s file has invalid format in line %s", DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ } /* @@ -1001,7 +1005,7 @@ dir_read_file_list(const char *root, const char *file_txt) char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; - uint64 write_size, + int64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, @@ -1016,12 +1020,7 @@ dir_read_file_list(const char *root, const char *file_txt) get_control_value(buf, "is_datafile", NULL, &is_datafile, true); get_control_value(buf, "is_cfs", NULL, &is_cfs, false); get_control_value(buf, "crc", NULL, &crc, true); - - /* optional fields */ - get_control_value(buf, "linked", linked, NULL, false); - get_control_value(buf, "segno", NULL, &segno, false); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "n_blocks", NULL, &n_blocks, false); if (root) join_path_components(filepath, root, path); @@ -1036,10 +1035,19 @@ dir_read_file_list(const char *root, const char *file_txt) file->is_cfs = is_cfs ? true : false; file->crc = (pg_crc32) crc; file->compress_alg = parse_compress_alg(compress_alg_string); - if (linked[0]) + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) file->linked = pgut_strdup(linked); - file->segno = (int) segno; - file->n_blocks = (int) n_blocks; + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; parray_append(files, file); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 55aa5766..b53f150c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -165,6 +165,7 @@ typedef enum ShowFormat /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) typedef struct pgBackupConfig { From 6d912340d655e5baa396cd31078222fec1a46f7b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 25 Jul 2018 14:17:04 +0300 Subject: [PATCH 87/99] PGPRO-427: Fix bug in SimpleXLogPageRead() with parallel mode --- src/parsexlog.c | 43 ++++++++++++++++++++++++--------- tests/helpers/ptrack_helpers.py | 25 ++++++++++++++----- tests/page.py | 40 +++++++++++++++++++++--------- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 71e6ddc5..927f3605 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -141,12 +141,13 @@ static void * doExtractPageMap(void *arg) { xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; XLogReaderState *xlogreader; XLogSegNo nextSegNo = 0; char *errormsg; - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, - &extract_arg->private_data); + private_data = &extract_arg->private_data; + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); if (xlogreader == NULL) elog(ERROR, "out of memory"); @@ -158,6 +159,9 @@ doExtractPageMap(void *arg) (uint32) (extract_arg->startpoint >> 32), (uint32) (extract_arg->startpoint)); + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; + do { XLogRecord *record; @@ -171,23 +175,28 @@ doExtractPageMap(void *arg) { XLogRecPtr errptr; - /* Try to switch to the next WAL segment */ - if (extract_arg->private_data.need_switch) + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) { - extract_arg->private_data.need_switch = false; + private_data->need_switch = false; + /* Critical section */ pthread_lock(&wal_segment_mutex); Assert(nextSegNoToRead); - extract_arg->private_data.xlogsegno = nextSegNoToRead; + private_data->xlogsegno = nextSegNoToRead; nextSegNoToRead++; pthread_mutex_unlock(&wal_segment_mutex); /* We reach the end */ - if (extract_arg->private_data.xlogsegno > extract_arg->endSegNo) + if (private_data->xlogsegno > extract_arg->endSegNo) break; /* Adjust next record position */ - XLogSegNoOffsetToRecPtr(extract_arg->private_data.xlogsegno, 0, + XLogSegNoOffsetToRecPtr(private_data->xlogsegno, 0, extract_arg->startpoint); /* Skip over the page header */ extract_arg->startpoint = XLogFindNextRecord(xlogreader, @@ -217,7 +226,7 @@ doExtractPageMap(void *arg) * start_lsn, we won't be able to build page map and PAGE backup will * be incorrect. Stop it and throw an error. */ - PrintXLogCorruptionMsg(&extract_arg->private_data, ERROR); + PrintXLogCorruptionMsg(private_data, ERROR); } extractPageInfo(xlogreader); @@ -255,8 +264,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, int threads_need = 0; XLogSegNo endSegNo; bool extract_isok = true; - pthread_t threads[num_threads]; - xlog_thread_arg thread_args[num_threads]; + pthread_t *threads; + xlog_thread_arg *thread_args; time_t start_time, end_time; @@ -276,6 +285,9 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, nextSegNoToRead = 0; time(&start_time); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + /* * Initialize thread args. * @@ -286,7 +298,6 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, { InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false); thread_args[i].thread_num = i; - thread_args[i].private_data.manual_switch = true; thread_args[i].startpoint = startpoint; thread_args[i].endpoint = endpoint; @@ -327,6 +338,9 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, extract_isok = false; } + pfree(threads); + pfree(thread_args); + time(&end_time); if (extract_isok) elog(LOG, "Pagemap compiled, time elapsed %.0f sec", @@ -700,6 +714,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (!XLByteInSeg(targetPagePtr, private_data->xlogsegno)) { CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ if (private_data->manual_switch) { private_data->need_switch = true; @@ -709,6 +727,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XLByteToSeg(targetPagePtr, private_data->xlogsegno); + /* Try to switch to the next WAL segment */ if (!private_data->xlogexists) { char xlogfname[MAXFNAMELEN]; diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index c430738c..3fd34de6 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -907,13 +907,26 @@ class ProbackupTest(object): return num def switch_wal_segment(self, node): - """ Execute pg_switch_wal/xlog() in given node""" - if self.version_to_num( - node.safe_psql("postgres", "show server_version") - ) >= self.version_to_num('10.0'): - node.safe_psql("postgres", "select pg_switch_wal()") + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") else: - node.safe_psql("postgres", "select pg_switch_xlog()") + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") sleep(1) def get_version(self, node): diff --git a/tests/page.py b/tests/page.py index 1ad1a75d..ef7122b6 100644 --- a/tests/page.py +++ b/tests/page.py @@ -528,9 +528,13 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "hot_standby": "on" } ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() self.set_archiving(backup_dir, 'node', node) node.start() @@ -542,13 +546,14 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.assertEqual(show_backup['backup-mode'], "FULL") # Fill instance with data and make several WAL segments ... - node.safe_psql("postgres", "create table test (id int)") - for x in range(0, 8): - node.safe_psql( - "postgres", - "insert into test select i from generate_series(1,100) s(i)") - self.switch_wal_segment(node) - count1 = node.safe_psql("postgres", "select count(*) from test") + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") # ... and do page backup with parallel pagemap self.backup_node( @@ -558,18 +563,29 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "PAGE") - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.start() + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() # Check restored node - count2 = node.safe_psql("postgres", "select count(*) from test") + count2 = node_restored.execute("postgres", "select count(*) from test") self.assertEqual(count1, count2) # Clean after yourself node.cleanup() + node_restored.cleanup() self.del_test_dir(module_name, fname) def test_parallel_pagemap_1(self): From 586e3366c7abd2e767e6ab91d42128c690fd8ba7 Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Wed, 20 Jun 2018 16:00:44 +0300 Subject: [PATCH 88/99] Add support of recovery target LSN --- src/help.c | 10 ++- src/parsexlog.c | 9 +- src/pg_probackup.c | 4 +- src/pg_probackup.h | 11 ++- src/restore.c | 34 +++++++- src/util.c | 29 +++++++ src/utils/pgut.c | 27 ++++++ src/utils/pgut.h | 2 + src/validate.c | 2 +- tests/cfs_restore.py | 2 +- tests/expected/option_help.out | 4 +- tests/restore_test.py | 147 +++++++++++++++++++++++++++++++++ 12 files changed, 265 insertions(+), 16 deletions(-) diff --git a/src/help.c b/src/help.c index 549a2e2b..e11f398f 100644 --- a/src/help.c +++ b/src/help.c @@ -116,7 +116,7 @@ help_pg_probackup(void) printf(_("\n %s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); @@ -125,7 +125,7 @@ help_pg_probackup(void) printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); @@ -265,7 +265,7 @@ help_restore(void) { printf(_("%s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); @@ -280,6 +280,7 @@ help_restore(void) printf(_(" --progress show progress\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); @@ -323,7 +324,7 @@ help_validate(void) { printf(_("%s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -333,6 +334,7 @@ help_validate(void) printf(_(" --progress show progress\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" --recovery-target-name=target-name\n")); diff --git a/src/parsexlog.c b/src/parsexlog.c index a0079b42..1410eb7c 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -299,6 +299,7 @@ validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; @@ -350,7 +351,7 @@ validate_wal(pgBackup *backup, * If recovery target is provided check that we can restore backup to a * recovery target time or xid. */ - if (!TransactionIdIsValid(target_xid) && target_time == 0) + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ elog(INFO, "Backup %s WAL segments are valid", backup_id); @@ -384,7 +385,8 @@ validate_wal(pgBackup *backup, last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) - || (target_time != 0 && backup->recovery_time >= target_time)) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) all_wal = true; startpoint = backup->stop_lsn; @@ -470,6 +472,9 @@ validate_wal(pgBackup *backup, else if (target_time != 0) elog(ERROR, "not enough WAL records to time %s", target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } /* clean */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index d1217e00..fdfdf5aa 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -61,6 +61,7 @@ uint32 replica_timeout = 300; /* default is 300 seconds */ /* restore options */ static char *target_time; static char *target_xid; +static char *target_lsn; static char *target_inclusive; static TimeLineID target_tli; static bool target_immediate; @@ -150,6 +151,7 @@ static pgut_option options[] = { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, /* delete options */ { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, @@ -436,7 +438,7 @@ main(int argc, char *argv[]) { /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_immediate, + target_inclusive, target_tli, target_lsn, target_immediate, target_name, target_action, restore_no_validate); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 55aa5766..2bf044a8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -258,6 +258,10 @@ typedef struct pgRecoveryTarget TransactionId recovery_target_xid; /* add one more field in order to avoid deparsing recovery_target_xid back */ const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; TimeLineID recovery_target_tli; bool recovery_target_inclusive; bool inclusive_specified; @@ -397,8 +401,9 @@ extern bool satisfy_recovery_target(const pgBackup *backup, extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, - const char *target_inclusive, TimeLineID target_tli, bool target_immediate, - const char *target_name, const char *target_action, bool restore_no_validate); + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); extern void opt_tablespace_map(pgut_option *opt, const char *arg); @@ -511,6 +516,7 @@ extern void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, @@ -533,6 +539,7 @@ extern long unsigned int base36dec(const char *text); extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern pg_time_t timestamptz_to_time_t(TimestampTz t); +extern int parse_server_version(char *server_version_str); extern void pgBackup_init(pgBackup *backup); /* in status.c */ diff --git a/src/restore.c b/src/restore.c index 165811bc..3ebe70d7 100644 --- a/src/restore.c +++ b/src/restore.c @@ -276,7 +276,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * because it's needed to form the name of xlog file. */ validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, base_full_backup->tli); + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli); /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ if (corrupted_backup) @@ -335,6 +336,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, for (i = base_full_backup_index; i >= dest_backup_index; i--) { pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + restore_backup(backup); } @@ -837,6 +843,9 @@ create_recovery_conf(time_t backup_id, if (rt->xid_specified) fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + if (rt->recovery_target_immediate) fprintf(fp, "recovery_target = 'immediate'\n"); @@ -981,6 +990,9 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) if (rt->time_specified) return backup->recovery_time <= rt->recovery_target_time; + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + return true; } @@ -1009,6 +1021,7 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, + const char *target_lsn, bool target_immediate, const char *target_name, const char *target_action, @@ -1017,6 +1030,7 @@ parseRecoveryTargetOptions(const char *target_time, time_t dummy_time; TransactionId dummy_xid; bool dummy_bool; + XLogRecPtr dummy_lsn; /* * count the number of the mutually exclusive options which may specify * recovery target. If final value > 1, throw an error. @@ -1028,10 +1042,13 @@ parseRecoveryTargetOptions(const char *target_time, rt->time_specified = false; rt->xid_specified = false; rt->inclusive_specified = false; + rt->lsn_specified = false; rt->recovery_target_time = 0; rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; rt->target_time_string = NULL; rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; rt->recovery_target_inclusive = false; rt->recovery_target_tli = 0; rt->recovery_target_immediate = false; @@ -1068,6 +1085,17 @@ parseRecoveryTargetOptions(const char *target_time, elog(ERROR, "Invalid value of --xid option %s", target_xid); } + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + if (target_inclusive) { rt->inclusive_specified = true; @@ -1112,10 +1140,10 @@ parseRecoveryTargetOptions(const char *target_time, /* More than one mutually exclusive option was defined. */ if (recovery_target_specified > 1) - elog(ERROR, "At most one of --immediate, --target-name, --time, or --xid can be used"); + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); /* If none of the options is defined, '--inclusive' option is meaningless */ - if (!(rt->xid_specified || rt->time_specified) && rt->recovery_target_inclusive) + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); return rt; diff --git a/src/util.c b/src/util.c index 051f4d54..0df94da6 100644 --- a/src/util.c +++ b/src/util.c @@ -235,6 +235,35 @@ timestamptz_to_time_t(TimestampTz t) return result; } +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + const char * status2str(BackupStatus status) { diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 330f2fa5..3ef77bdc 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -31,6 +31,7 @@ #define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ #define SECS_PER_MINUTE 60 #define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 const char *PROGRAM_NAME = NULL; @@ -983,6 +984,32 @@ parse_int(const char *value, int *result, int flags, const char **hintmsg) return true; } +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + static char * longopts_to_optstring(const struct option opts[], const size_t len) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 662f07c1..fdec956d 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -17,6 +17,7 @@ #include #include +#include "access/xlogdefs.h" #include "logger.h" #if !defined(C_H) && !defined(__cplusplus) @@ -207,6 +208,7 @@ extern bool parse_uint64(const char *value, uint64 *result, int flags); extern bool parse_time(const char *value, time_t *result, bool utc_default); extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); extern void convert_from_base_unit(int64 base_value, int base_unit, int64 *value, const char **unit); diff --git a/src/validate.c b/src/validate.c index 41467f25..139ae679 100644 --- a/src/validate.c +++ b/src/validate.c @@ -328,7 +328,7 @@ do_validate_instance(void) base36enc(current_backup->start_time)); /* Validate corresponding WAL files */ validate_wal(current_backup, arclog_path, 0, - 0, base_full_backup->tli); + 0, 0, base_full_backup->tli); } /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ if (current_backup->status == BACKUP_STATUS_CORRUPT) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 2004cb14..73553a30 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -4,7 +4,7 @@ restore pg_probackup restore -B backupdir --instance instance_name [-D datadir] - [ -i backup_id | [{--time=time | --xid=xid } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] [-j num_threads] [--progress] [-q] [-v] """ diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index f9c59e0e..d60d4d08 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -53,7 +53,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup restore -B backup-dir --instance=instance_name [-D pgdata-dir] [-i backup-id] [--progress] - [--time=time|--xid=xid [--inclusive=boolean]] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--timeline=timeline] [-T OLDDIR=NEWDIR] [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] @@ -62,7 +62,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup validate -B backup-dir [--instance=instance_name] [-i backup-id] [--progress] - [--time=time|--xid=xid [--inclusive=boolean]] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--recovery-target-name=target-name] [--timeline=timeline] diff --git a/tests/restore_test.py b/tests/restore_test.py index 862c8662..c33a1e29 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -353,6 +353,153 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" From ab86cb5962038c14a7e6c2a0b5ce7b125b65589e Mon Sep 17 00:00:00 2001 From: Aleksandr Parfenov Date: Sat, 9 Jun 2018 14:59:35 +0300 Subject: [PATCH 89/99] Warning in case of empty/corrupted backup.control file --- src/catalog.c | 21 +++++++++- src/utils/pgut.c | 17 ++++++--- src/utils/pgut.h | 2 +- tests/option_test.py | 5 ++- tests/show_test.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 9 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index a0a8e5dc..78630b85 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -479,6 +479,7 @@ readBackupControlFile(const char *path) char *program_version = NULL; char *server_version = NULL; char *compress_alg = NULL; + int parsed_options; pgut_option options[] = { @@ -508,10 +509,28 @@ readBackupControlFile(const char *path) }; if (access(path, F_OK) != 0) + { + elog(WARNING, "control file \"%s\" doesn't exist", path); + pgBackupFree(backup); return NULL; + } pgBackup_init(backup); - pgut_readopt(path, options, ERROR); + parsed_options = pgut_readopt(path, options, WARNING); + + if (parsed_options == 0) + { + elog(WARNING, "control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } if (backup_mode) { diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 330f2fa5..6948a0c1 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1095,20 +1095,22 @@ key_equals(const char *lhs, const char *rhs) /* * Get configuration from configuration file. + * Return number of parsed options */ -void +int pgut_readopt(const char *path, pgut_option options[], int elevel) { FILE *fp; char buf[1024]; char key[1024]; char value[1024]; + int parsed_options = 0; if (!options) - return; + return parsed_options; if ((fp = pgut_fopen(path, "rt", true)) == NULL) - return; + return parsed_options; while (fgets(buf, lengthof(buf), fp)) { @@ -1127,18 +1129,23 @@ pgut_readopt(const char *path, pgut_option options[], int elevel) { if (opt->allowed < SOURCE_FILE && opt->allowed != SOURCE_FILE_STRICT) - elog(elevel, "option %s cannot specified in file", opt->lname); + elog(elevel, "option %s cannot be specified in file", opt->lname); else if (opt->source <= SOURCE_FILE) + { assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } break; } } if (!options[i].type) - elog(elevel, "invalid option \"%s\"", key); + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); } } fclose(fp); + + return parsed_options; } static const char * diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 662f07c1..5e447a07 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -110,7 +110,7 @@ extern bool in_cleanup; extern bool in_password; /* User prompts password */ extern int pgut_getopt(int argc, char **argv, pgut_option options[]); -extern void pgut_readopt(const char *path, pgut_option options[], int elevel); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel); extern void pgut_getopt_env(pgut_option options[]); extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); diff --git a/tests/option_test.py b/tests/option_test.py index a66d346e..34f7cd0f 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -199,7 +199,8 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) # invalid option in pg_probackup.conf - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: conf.write("TIMELINEID=1\n") try: @@ -209,7 +210,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: invalid option "TIMELINEID"\n', + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself diff --git a/tests/show_test.py b/tests/show_test.py index 29d0bdb3..931da184 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -110,3 +110,94 @@ class OptionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 264cd18fb449562ea1385f7d57a065445d268e5e Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 30 Jul 2018 16:31:43 +0300 Subject: [PATCH 90/99] check links to parent backup in validate --- src/catalog.c | 45 +++++++++++++++++++++++++++++++++++++++ src/pg_probackup.h | 2 ++ src/restore.c | 53 +++++++++++++++++++++++----------------------- src/validate.c | 43 ++++++++++++++----------------------- 4 files changed, 89 insertions(+), 54 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 45e080c2..e47b9899 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -714,3 +714,48 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, make_native_path(path); } + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f6eae19f..92693884 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -443,6 +443,8 @@ extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); +extern pgBackup* find_parent_backup(pgBackup *current_backup); + /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root); diff --git a/src/restore.c b/src/restore.c index 52378168..5db9589d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -202,41 +202,40 @@ do_restore_or_validate(time_t target_backup_id, dest_backup_index = i; } - /* If we already found dest_backup, look for full backup. */ - /* TODO Now, as we have all backups linked, we can probably get rid of that?"*/ - if (dest_backup) + } + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) { - if (current_backup->backup_mode == BACKUP_MODE_FULL) + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) + { + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) { - if (current_backup->status != BACKUP_STATUS_OK) - { - /* Full backup revalidation can be done only for DONE and CORRUPT */ - if (current_backup->status == BACKUP_STATUS_DONE || - current_backup->status == BACKUP_STATUS_CORRUPT) - elog(WARNING, "base backup %s for given backup %s is in %s status, trying to revalidate", - base36enc_dup(current_backup->start_time), - base36enc_dup(dest_backup->start_time), - status2str(current_backup->status)); - else - elog(ERROR, "base backup %s for given backup %s is in %s status", - base36enc_dup(current_backup->start_time), - base36enc_dup(dest_backup->start_time), - status2str(current_backup->status)); - } - /* We found both dest and base backups. */ - base_full_backup = current_backup; base_full_backup_index = i; break; } - else - /* It`s ok to skip incremental backup */ - continue; } } - if (base_full_backup == NULL) - elog(ERROR, "Full backup satisfying target options is not found."); - /* * Ensure that directories provided in tablespace mapping are valid * i.e. empty or not exist. diff --git a/src/validate.c b/src/validate.c index b4a1c839..ec124a9e 100644 --- a/src/validate.c +++ b/src/validate.c @@ -282,44 +282,33 @@ do_validate_instance(void) if (backups == NULL) elog(ERROR, "Failed to get backup list."); - /* Valiate each backup along with its xlog files. */ - /* TODO Maybe use parent_backup_link instead of looking for backups in the list */ + /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) { - pgBackup *base_full_backup = NULL; - current_backup = (pgBackup *) parray_get(backups, i); - if (current_backup->backup_mode != BACKUP_MODE_FULL) - { - int j; - - for (j = i + 1; j < parray_num(backups); j++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (backup->backup_mode == BACKUP_MODE_FULL) - { - base_full_backup = backup; - break; - } - } - } - else - base_full_backup = current_backup; - + /* Valiate each backup along with its xlog files. */ pgBackupValidate(current_backup); - /* There is no point in wal validation for corrupted backup */ + /* Ensure that the backup has valid list of parent backups */ if (current_backup->status == BACKUP_STATUS_OK) { - if (base_full_backup == NULL) - elog(ERROR, "Valid full backup for backup %s is not found.", - base36enc(current_backup->start_time)); + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + /* Validate corresponding WAL files */ validate_wal(current_backup, arclog_path, 0, - 0, base_full_backup->tli); + 0, base_full_backup->tli); } + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ if (current_backup->status == BACKUP_STATUS_CORRUPT) { From e7d088c238de36120508576064a646c243bf3f75 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 1 Aug 2018 16:48:37 +0300 Subject: [PATCH 91/99] PGPRO-1700: Show default options in show-config and clean up the code of the config options. --- src/configure.c | 192 +++++++++++++++++++++++---------------------- src/delete.c | 14 ++-- src/help.c | 8 +- src/pg_probackup.c | 13 +-- src/pg_probackup.h | 14 +++- src/restore.c | 8 +- src/util.c | 4 +- src/utils/logger.c | 23 +++--- src/utils/logger.h | 6 +- src/utils/pgut.c | 6 +- 10 files changed, 159 insertions(+), 129 deletions(-) diff --git a/src/configure.c b/src/configure.c index 11c068a5..bb67ec5f 100644 --- a/src/configure.c +++ b/src/configure.c @@ -8,6 +8,7 @@ */ #include "pg_probackup.h" +#include "utils/logger.h" #include "pqexpbuffer.h" @@ -57,13 +58,17 @@ do_configure(bool show_only) config->master_db = master_db; if (master_user) config->master_user = master_user; - if (replica_timeout != 300) /* 300 is default value */ + + if (replica_timeout) config->replica_timeout = replica_timeout; - if (log_level_console != LOG_NONE) - config->log_level_console = LOG_LEVEL_CONSOLE; - if (log_level_file != LOG_NONE) - config->log_level_file = LOG_LEVEL_FILE; + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; if (log_filename) config->log_filename = log_filename; if (error_log_filename) @@ -80,9 +85,9 @@ do_configure(bool show_only) if (retention_window) config->retention_window = retention_window; - if (compress_alg != NOT_DEFINED_COMPRESS) + if (compress_alg) config->compress_alg = compress_alg; - if (compress_level != DEFAULT_COMPRESS_LEVEL) + if (compress_level) config->compress_level = compress_level; if (show_only) @@ -107,21 +112,23 @@ pgBackupConfigInit(pgBackupConfig *config) config->master_port = NULL; config->master_db = NULL; config->master_user = NULL; - config->replica_timeout = INT_MIN; /* INT_MIN means "undefined" */ + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - config->log_level_console = INT_MIN; /* INT_MIN means "undefined" */ - config->log_level_file = INT_MIN; /* INT_MIN means "undefined" */ - config->log_filename = NULL; + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; config->error_log_filename = NULL; - config->log_directory = NULL; - config->log_rotation_size = 0; - config->log_rotation_age = 0; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; - config->retention_redundancy = 0; - config->retention_window = 0; + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; - config->compress_alg = NOT_DEFINED_COMPRESS; - config->compress_level = DEFAULT_COMPRESS_LEVEL; + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; } void @@ -154,55 +161,43 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) if (config->master_user) fprintf(out, "master-user = %s\n", config->master_user); - if (config->replica_timeout != INT_MIN) - { - convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, - &res, &unit); - fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); - } + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); fprintf(out, "#Logging parameters:\n"); - if (config->log_level_console != INT_MIN) - fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); - if (config->log_level_file != INT_MIN) - fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); - if (config->log_filename) - fprintf(out, "log-filename = %s\n", config->log_filename); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); if (config->error_log_filename) fprintf(out, "error-log-filename = %s\n", config->error_log_filename); - if (config->log_directory) - fprintf(out, "log-directory = %s\n", config->log_directory); - /* - * Convert values from base unit - */ - if (config->log_rotation_size) - { - convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, - &res, &unit); - fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, unit); - } - if (config->log_rotation_age) - { - convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, - &res, &unit); - fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, unit); - } + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); fprintf(out, "#Retention parameters:\n"); - if (config->retention_redundancy) - fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); - if (config->retention_window) - fprintf(out, "retention-window = %u\n", config->retention_window); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); fprintf(out, "#Compression parameters:\n"); fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); - - if (compress_level != config->compress_level) - fprintf(out, "compress-level = %d\n", compress_level); - else - fprintf(out, "compress-level = %d\n", config->compress_level); + fprintf(out, "compress-level = %d\n", config->compress_level); } void @@ -258,6 +253,8 @@ readBackupCatalogConfigFile(void) { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, /* other options */ { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, {0} }; @@ -342,6 +339,8 @@ static void show_configure_json(pgBackupConfig *config) { PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; json_add(buf, JT_BEGIN_OBJECT, &json_level); @@ -373,53 +372,60 @@ show_configure_json(pgBackupConfig *config) json_add_value(buf, "master-user", config->master_user, json_level, true); - if (config->replica_timeout != INT_MIN) - { - json_add_key(buf, "replica-timeout", json_level, true); - appendPQExpBuffer(buf, "%d", config->replica_timeout); - } + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); /* Logging parameters */ - if (config->log_level_console != INT_MIN) - json_add_value(buf, "log-level-console", - deparse_log_level(config->log_level_console), json_level, - true); - if (config->log_level_file != INT_MIN) - json_add_value(buf, "log-level-file", - deparse_log_level(config->log_level_file), json_level, - true); - if (config->log_filename) - json_add_value(buf, "log-filename", config->log_filename, json_level, - true); + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); if (config->error_log_filename) json_add_value(buf, "error-log-filename", config->error_log_filename, json_level, true); - if (config->log_directory) - json_add_value(buf, "log-directory", config->log_directory, json_level, - true); - if (config->log_rotation_size) + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) { - json_add_key(buf, "log-rotation-size", json_level, true); - appendPQExpBuffer(buf, "%d", config->log_rotation_size); - } - if (config->log_rotation_age) - { - json_add_key(buf, "log-rotation-age", json_level, true); - appendPQExpBuffer(buf, "%d", config->log_rotation_age); + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); /* Retention parameters */ - if (config->retention_redundancy) - { - json_add_key(buf, "retention-redundancy", json_level, true); - appendPQExpBuffer(buf, "%u", config->retention_redundancy); - } - if (config->retention_window) - { - json_add_key(buf, "retention-window", json_level, true); - appendPQExpBuffer(buf, "%u", config->retention_window); - } + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); /* Compression parameters */ json_add_value(buf, "compress-algorithm", diff --git a/src/delete.c b/src/delete.c index c933382e..09cd4e0f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -141,7 +141,8 @@ do_retention_purge(void) if (retention_window > 0) elog(LOG, "WINDOW=%u", retention_window); - if (retention_redundancy == 0 && retention_window == 0) + if (retention_redundancy == 0 + && retention_window == 0) { elog(WARNING, "Retention policy is not set"); if (!delete_wal) @@ -161,7 +162,8 @@ do_retention_purge(void) } /* Find target backups to be deleted */ - if (delete_expired && (retention_redundancy > 0 || retention_window > 0)) + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) { backup_num = 0; for (i = 0; i < parray_num(backup_list); i++) @@ -173,13 +175,13 @@ do_retention_purge(void) if (backup->status != BACKUP_STATUS_OK) continue; /* - * When a validate full backup was found, we can delete the + * When a valid full backup was found, we can delete the * backup that is older than it using the number of generations. */ if (backup->backup_mode == BACKUP_MODE_FULL) backup_num++; - /* Evaluateretention_redundancy if this backup is eligible for removal */ + /* Evaluate retention_redundancy if this backup is eligible for removal */ if (keep_next_backup || retention_redundancy >= backup_num_evaluate + 1 || (retention_window > 0 && backup->recovery_time >= days_threshold)) @@ -200,6 +202,7 @@ do_retention_purge(void) continue; } + /* Delete backup and update status to DELETED */ pgBackupDeleteFiles(backup); backup_deleted = true; @@ -259,7 +262,8 @@ pgBackupDeleteFiles(pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); /* * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which diff --git a/src/help.c b/src/help.c index e11f398f..d8726de3 100644 --- a/src/help.c +++ b/src/help.c @@ -87,6 +87,7 @@ help_pg_probackup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); @@ -257,7 +258,7 @@ help_backup(void) printf(_(" --master-db=db_name database to connect to master\n")); printf(_(" --master-host=host_name database server host of master\n")); printf(_(" --master-port=port database server port of master\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication in seconds\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); } static void @@ -429,6 +430,7 @@ help_set_config(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -477,7 +479,9 @@ help_set_config(void) printf(_(" --master-db=db_name database to connect to master\n")); printf(_(" --master-host=host_name database server host of master\n")); printf(_(" --master-port=port database server port of master\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fdfdf5aa..df143936 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -51,12 +51,12 @@ bool backup_logs = false; bool smooth_checkpoint; bool is_remote_backup = false; /* Wait timeout for WAL segment archiving */ -uint32 archive_timeout = 300; /* default is 300 seconds */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; const char *master_db = NULL; const char *master_host = NULL; const char *master_port= NULL; const char *master_user = NULL; -uint32 replica_timeout = 300; /* default is 300 seconds */ +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; /* restore options */ static char *target_time; @@ -84,9 +84,10 @@ uint32 retention_redundancy = 0; uint32 retention_window = 0; /* compression options */ -CompressAlg compress_alg = NOT_DEFINED_COMPRESS; -int compress_level = DEFAULT_COMPRESS_LEVEL; -bool compress_shortcut = false; +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; +bool compress_shortcut = false; + /* other options */ char *instance_name; @@ -571,7 +572,7 @@ compress_init(void) if (backup_subcmd != SET_CONFIG_CMD) { - if (compress_level != DEFAULT_COMPRESS_LEVEL + if (compress_level != COMPRESS_LEVEL_DEFAULT && compress_alg == NOT_DEFINED_COMPRESS) elog(ERROR, "Cannot specify compress-level option without compress-alg option"); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 25314f10..04683e65 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -63,6 +63,8 @@ #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" /* Direcotry/File permission */ #define DIR_PERMISSION (0700) #define FILE_PERMISSION (0600) @@ -182,6 +184,8 @@ typedef struct pgBackupConfig const char *master_user; int replica_timeout; + int archive_timeout; + int log_level_console; int log_level_file; char *log_filename; @@ -334,12 +338,14 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 extern uint32 archive_timeout; extern bool is_remote_backup; extern const char *master_db; extern const char *master_host; extern const char *master_port; extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 extern uint32 replica_timeout; extern bool is_ptrack_support; @@ -355,7 +361,10 @@ extern bool delete_expired; extern bool apply_to_all; extern bool force_delete; -/* retention options */ +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + extern uint32 retention_redundancy; extern uint32 retention_window; @@ -364,7 +373,8 @@ extern CompressAlg compress_alg; extern int compress_level; extern bool compress_shortcut; -#define DEFAULT_COMPRESS_LEVEL 1 +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); diff --git a/src/restore.c b/src/restore.c index 02296c2c..2a0fec83 100644 --- a/src/restore.c +++ b/src/restore.c @@ -14,8 +14,10 @@ #include #include #include +#include #include "catalog/pg_control.h" +#include "utils/logger.h" #include "utils/thread.h" typedef struct @@ -455,7 +457,7 @@ restore_backup(pgBackup *backup) parray_walk(files, pgFileFree); parray_free(files); - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) + if (log_level_console <= LOG || log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); } @@ -492,7 +494,7 @@ remove_deleted_files(pgBackup *backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) + if (log_level_console <= LOG || log_level_file <= LOG) elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); } } @@ -667,7 +669,7 @@ check_tablespace_mapping(pgBackup *backup) pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); read_tablespace_map(links, this_backup_path); - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) + if (log_level_console <= LOG || log_level_file <= LOG) elog(LOG, "check tablespace directories of backup %s", base36enc(backup->start_time)); diff --git a/src/util.c b/src/util.c index 0df94da6..798e3a1f 100644 --- a/src/util.c +++ b/src/util.c @@ -341,8 +341,8 @@ pgBackup_init(pgBackup *backup) backup->data_bytes = BYTES_INVALID; backup->wal_bytes = BYTES_INVALID; - backup->compress_alg = NOT_DEFINED_COMPRESS; - backup->compress_level = 0; + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; backup->block_size = BLCKSZ; backup->wal_block_size = XLOG_BLCKSZ; diff --git a/src/utils/logger.c b/src/utils/logger.c index 36cffb20..88ee9b6b 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -15,12 +15,13 @@ #include "logger.h" #include "pgut.h" +#include "pg_probackup.h" #include "thread.h" /* Logger parameters */ -int log_level_console = LOG_NONE; -int log_level_file = LOG_NONE; +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; char *log_filename = NULL; char *error_log_filename = NULL; @@ -74,12 +75,12 @@ void init_logger(const char *root_path) { /* Set log path */ - if (LOG_LEVEL_FILE != LOG_OFF || error_log_filename) + if (log_level_file != LOG_OFF || error_log_filename) { if (log_directory) strcpy(log_path, log_directory); else - join_path_components(log_path, root_path, "log"); + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); } } @@ -165,10 +166,10 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) time_t log_time = (time_t) time(NULL); char strfbuf[128]; - write_to_file = elevel >= LOG_LEVEL_FILE && log_path[0] != '\0'; + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; write_to_error_log = elevel >= ERROR && error_log_filename && log_path[0] != '\0'; - write_to_stderr = elevel >= LOG_LEVEL_CONSOLE && !file_only; + write_to_stderr = elevel >= log_level_console && !file_only; pthread_lock(&log_file_mutex); #ifdef WIN32 @@ -201,7 +202,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (log_file == NULL) { if (log_filename == NULL) - open_logfile(&log_file, "pg_probackup.log"); + open_logfile(&log_file, LOG_FILENAME_DEFAULT); else open_logfile(&log_file, log_filename); } @@ -271,7 +272,7 @@ elog_stderr(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < ERROR) + if (elevel < log_level_console && elevel < ERROR) return; va_start(args, fmt); @@ -298,7 +299,7 @@ elog(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -318,7 +319,7 @@ elog_file(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -359,7 +360,7 @@ pg_log(eLogType type, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); diff --git a/src/utils/logger.h b/src/utils/logger.h index 315c337a..0177c551 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -36,11 +36,13 @@ extern char *error_log_filename; extern char *log_directory; extern char log_path[MAXPGPATH]; +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 extern int log_rotation_size; extern int log_rotation_age; -#define LOG_LEVEL_CONSOLE ((log_level_console == LOG_NONE) ? INFO : log_level_console) -#define LOG_LEVEL_FILE ((log_level_file == LOG_NONE) ? LOG_OFF : log_level_file) +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF #undef elog extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index c062aec8..37f249a2 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1680,7 +1680,7 @@ pgut_execute_parallel(PGconn* conn, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; @@ -1742,7 +1742,7 @@ pgut_execute_extended(PGconn* conn, const char *query, int nParams, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; @@ -1800,7 +1800,7 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; From 4ed28cf6e5721e16337b36cfd9aa58837c3b6eb1 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 2 Aug 2018 11:57:39 +0300 Subject: [PATCH 92/99] PGPRO-1449: Add MERGE command: merge PAGE with FULL backup --- Makefile | 2 +- src/backup.c | 70 ++--- src/catalog.c | 91 +++++- src/data.c | 124 +++++--- src/delete.c | 2 - src/dir.c | 393 +++++++++++++++++++++++- src/help.c | 18 +- src/merge.c | 520 ++++++++++++++++++++++++++++++++ src/pg_probackup.c | 49 +-- src/pg_probackup.h | 45 ++- src/restore.c | 379 +---------------------- src/show.c | 2 - src/util.c | 36 +-- src/validate.c | 4 +- tests/expected/option_help.out | 4 +- tests/helpers/ptrack_helpers.py | 10 + tests/merge.py | 454 ++++++++++++++++++++++++++++ tests/option_test.py | 1 + 18 files changed, 1657 insertions(+), 547 deletions(-) create mode 100644 src/merge.c create mode 100644 tests/merge.py diff --git a/Makefile b/Makefile index eb3e3ee2..e22bc86c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ - src/utils/json.o src/utils/thread.o + src/utils/json.o src/utils/thread.o src/merge.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h diff --git a/src/backup.c b/src/backup.c index 2c0fd451..09fe7cd6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -104,7 +104,6 @@ static int checkpoint_timeout(void); //static void backup_list_file(parray *files, const char *root, ) static void parse_backup_filelist_filenames(parray *files, const char *root); -static void write_backup_file_list(parray *files, const char *root); static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); @@ -515,8 +514,6 @@ do_backup_instance(void) /* get list of backups already taken */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); prev_backup = catalog_get_last_data_backup(backup_list, current.tli); if (prev_backup == NULL) @@ -697,8 +694,11 @@ do_backup_instance(void) pg_atomic_clear_flag(&file->lock); } - /* sort by size for load balancing */ + /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -788,7 +788,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - write_backup_file_list(backup_files_list, pgdata); + pgBackupWriteFileList(¤t, backup_files_list, pgdata); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -2076,35 +2076,32 @@ backup_files(void *arg) /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) { - int p; char *relative; - int n_prev_files = parray_num(arguments->prev_filelist); + pgFile key; + pgFile **prev_file; relative = GetRelativePath(file->path, arguments->from_root); - for (p = 0; p < n_prev_files; p++) - { - pgFile *prev_file; + key.path = relative; - prev_file = (pgFile *) parray_get(arguments->prev_filelist, p); - - if (strcmp(relative, prev_file->path) == 0) - { - /* File exists in previous backup */ - file->exists_in_prev = true; - // elog(VERBOSE, "File exists at the time of previous backup %s", relative); - break; - } - } + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; } /* copy the file into backup */ if (file->is_datafile && !file->is_cfs) { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + /* backup block by block if datafile AND not compressed by cfs*/ - if (!backup_data_file(arguments, - arguments->from_root, - arguments->to_root, file, + if (!backup_data_file(arguments, to_path, file, arguments->prev_start_lsn, - current.backup_mode)) + current.backup_mode, + compress_alg, compress_level)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); @@ -2279,31 +2276,6 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) free(cfs_tblspc_path); } - -/* - * Output the list of files to backup catalog DATABASE_FILE_LIST - */ -static void -write_backup_file_list(parray *files, const char *root) -{ - FILE *fp; - char path[MAXPGPATH]; - - pgBackupGetPath(¤t, path, lengthof(path), DATABASE_FILE_LIST); - - fp = fopen(path, "wt"); - if (fp == NULL) - elog(ERROR, "cannot open file list \"%s\": %s", path, - strerror(errno)); - - print_file_list(fp, files, root); - - if (fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) - elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); -} - /* * Find pgfile by given rnode in the backup_files_list * and add given blkno to its pagemap. diff --git a/src/catalog.c b/src/catalog.c index 78630b85..2c4bff4a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -325,6 +325,9 @@ err_proc: if (backups) parray_walk(backups, pgBackupFree); parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + return NULL; } @@ -389,7 +392,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) deparse_compress_alg(backup->compress_alg)); fprintf(out, "compress-level = %d\n", backup->compress_level); fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); - + fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); @@ -462,6 +465,30 @@ pgBackupWriteBackupControlFile(pgBackup *backup) fclose(fp); } +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + /* * Read BACKUP_CONTROL_FILE and create pgBackup. * - Comment starts with ';'. @@ -515,7 +542,7 @@ readBackupControlFile(const char *path) return NULL; } - pgBackup_init(backup); + pgBackupInit(backup); parsed_options = pgut_readopt(path, options, WARNING); if (parsed_options == 0) @@ -566,10 +593,12 @@ readBackupControlFile(const char *path) { if (strcmp(status, "OK") == 0) backup->status = BACKUP_STATUS_OK; - else if (strcmp(status, "RUNNING") == 0) - backup->status = BACKUP_STATUS_RUNNING; else if (strcmp(status, "ERROR") == 0) backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; else if (strcmp(status, "DELETING") == 0) backup->status = BACKUP_STATUS_DELETING; else if (strcmp(status, "DELETED") == 0) @@ -698,11 +727,63 @@ deparse_compress_alg(int alg) return NULL; } +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + /* free pgBackup object */ void pgBackupFree(void *backup) { - free(backup); + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); } /* Compare two pgBackup with their IDs (start time) in ascending order */ diff --git a/src/data.c b/src/data.c index 1a84ea77..a66770bc 100644 --- a/src/data.c +++ b/src/data.c @@ -27,18 +27,24 @@ #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ -static int32 zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) { - uLongf compressed_size = dst_size; - int rc = compress2(dst, &compressed_size, src, src_size, compress_level); + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + return rc == Z_OK ? compressed_size : rc; } /* Implementation of zlib compression method */ -static int32 zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) { - uLongf dest_len = dst_size; - int rc = uncompress(dst, &dest_len, src, src_size); + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + return rc == Z_OK ? dest_len : rc; } #endif @@ -48,7 +54,8 @@ static int32 zlib_decompress(void* dst, size_t dst_size, void const* src, size_t * written in the destination buffer, or -1 if compression fails. */ static int32 -do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) { switch (alg) { @@ -57,7 +64,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_compress(dst, dst_size, src, src_size); + return zlib_compress(dst, dst_size, src, src_size, level); #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -71,7 +78,8 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre * decompressed in the destination buffer, or -1 if decompression fails. */ static int32 -do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) { switch (alg) { @@ -350,7 +358,8 @@ prepare_page(backup_files_arg *arguments, static void compress_and_backup_page(pgFile *file, BlockNumber blknum, FILE *in, FILE *out, pg_crc32 *crc, - int page_state, Page page) + int page_state, Page page, + CompressAlg calg, int clevel) { BackupPageHeader header; size_t write_buffer_size = sizeof(header); @@ -375,9 +384,9 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, { /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, compress_alg); + page, BLCKSZ, calg, clevel); - file->compress_alg = compress_alg; + file->compress_alg = calg; file->read_size += BLCKSZ; Assert (header.compressed_size <= BLCKSZ); @@ -386,7 +395,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, { memcpy(write_buffer, &header, sizeof(header)); memcpy(write_buffer + sizeof(header), - compressed_page, header.compressed_size); + compressed_page, header.compressed_size); write_buffer_size += MAXALIGN(header.compressed_size); } /* Nonpositive value means that compression failed. Write it as is. */ @@ -408,11 +417,12 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; + fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", - file->path, blknum, strerror(errno_tmp)); + file->path, blknum, strerror(errno_tmp)); } file->write_size += write_buffer_size; @@ -428,11 +438,10 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, */ bool backup_data_file(backup_files_arg* arguments, - const char *from_root, const char *to_root, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - BackupMode backup_mode) + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) { - char to_path[MAXPGPATH]; FILE *in; FILE *out; BlockNumber blknum = 0; @@ -499,7 +508,6 @@ backup_data_file(backup_files_arg* arguments, nblocks = file->size/BLCKSZ; /* open backup file for write */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, PG_BINARY_W); if (out == NULL) { @@ -522,10 +530,10 @@ backup_data_file(backup_files_arg* arguments, for (blknum = 0; blknum < nblocks; blknum++) { page_state = prepare_page(arguments, file, prev_backup_start_lsn, - blknum, nblocks, in, &n_blocks_skipped, + blknum, nblocks, in, &n_blocks_skipped, backup_mode, curr_page); compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page); + page_state, curr_page, calg, clevel); n_blocks_read++; if (page_state == PageIsTruncated) break; @@ -545,10 +553,10 @@ backup_data_file(backup_files_arg* arguments, while (datapagemap_next(iter, &blknum)) { page_state = prepare_page(arguments, file, prev_backup_start_lsn, - blknum, nblocks, in, &n_blocks_skipped, + blknum, nblocks, in, &n_blocks_skipped, backup_mode, curr_page); compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page); + page_state, curr_page, calg, clevel); n_blocks_read++; if (page_state == PageIsTruncated) break; @@ -595,14 +603,14 @@ backup_data_file(backup_files_arg* arguments, /* * Restore files in the from_root directory to the to_root directory with * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. */ void -restore_data_file(const char *from_root, - const char *to_root, - pgFile *file, - pgBackup *backup) +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) { - char to_path[MAXPGPATH]; FILE *in = NULL; FILE *out = NULL; BackupPageHeader header; @@ -627,7 +635,6 @@ restore_data_file(const char *from_root, * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, PG_BINARY_R "+"); if (out == NULL && errno == ENOENT) out = fopen(to_path, PG_BINARY_W); @@ -641,6 +648,7 @@ restore_data_file(const char *from_root, while (true) { + off_t write_pos; size_t read_len; DataPage compressed_page; /* used as read buffer */ DataPage page; @@ -682,7 +690,7 @@ restore_data_file(const char *from_root, if (header.block < blknum) elog(ERROR, "backup is broken at file->path %s block %u", - file->path, blknum); + file->path, blknum); blknum = header.block; @@ -707,37 +715,47 @@ restore_data_file(const char *from_root, if (header.compressed_size != BLCKSZ) { - size_t uncompressed_size = 0; + int32 uncompressed_size = 0; uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, - file->compress_alg); + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); if (uncompressed_size != BLCKSZ) - elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", - uncompressed_size); + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); } + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + /* * Seek and write the restored page. */ - if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0) + if (fseek(out, write_pos, SEEK_SET) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + if (header.compressed_size < BLCKSZ) { if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, file->path, strerror(errno)); } else { /* if page wasn't compressed, we've read full block */ if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, file->path, strerror(errno)); } } @@ -748,8 +766,7 @@ restore_data_file(const char *from_root, * So when restoring file from DELTA backup we, knowning it`s size at * a time of a backup, can truncate file to this size. */ - if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA && - file->n_blocks != BLOCKNUM_INVALID && !need_truncate) + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) { size_t file_size = 0; @@ -766,10 +783,15 @@ restore_data_file(const char *from_root, if (need_truncate) { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + /* * Truncate file to this length. */ - if (ftruncate(fileno(out), truncate_from * BLCKSZ) != 0) + if (ftruncate(fileno(out), write_pos) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); elog(INFO, "Delta truncate file %s to block %u", @@ -928,6 +950,22 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + #ifdef HAVE_LIBZ /* * Show error during work with compressed file diff --git a/src/delete.c b/src/delete.c index 09cd4e0f..0829e725 100644 --- a/src/delete.c +++ b/src/delete.c @@ -33,8 +33,6 @@ do_delete(time_t backup_id) /* Get complete list of backups */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); if (backup_id != 0) { diff --git a/src/dir.c b/src/dir.c index 370e1a27..a08bd934 100644 --- a/src/dir.c +++ b/src/dir.c @@ -87,6 +87,34 @@ static char *pgdata_exclude_files_non_exclusive[] = NULL }; +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); @@ -94,6 +122,13 @@ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list); +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + /* * Create directory, also create parent directories if necessary. */ @@ -225,7 +260,7 @@ delete_file: } pg_crc32 -pgFileGetCRC(pgFile *file) +pgFileGetCRC(const char *file_path) { FILE *fp; pg_crc32 crc = 0; @@ -234,10 +269,10 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, PG_BINARY_R); + fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", - file->path, strerror(errno)); + file_path, strerror(errno)); /* calc CRC of backup file */ INIT_CRC32C(crc); @@ -249,7 +284,7 @@ pgFileGetCRC(pgFile *file) } errno_tmp = errno; if (!feof(fp)) - elog(WARNING, "cannot read \"%s\": %s", file->path, + elog(WARNING, "cannot read \"%s\": %s", file_path, strerror(errno_tmp)); if (len > 0) COMP_CRC32C(crc, buf, len); @@ -677,7 +712,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, * **is_root** is a little bit hack. We exclude only first level of directories * and on the first level we check all files and directories. */ -void +static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude) { @@ -749,6 +784,277 @@ list_data_directories(parray *files, const char *path, bool is_root, path, strerror(prev_errno)); } +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + /* * Read names of symbolik names of tablespaces with links to directories from * tablespace_map or tablespace_map.txt. @@ -800,6 +1106,70 @@ read_tablespace_map(parray *files, const char *backup_dir) fclose(fp); } +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + /* * Print backup content list. */ @@ -858,7 +1228,7 @@ print_file_list(FILE *out, const parray *files, const char *root) * {"name1":"value1", "name2":"value2"} * * The value will be returned to "value_str" as string if it is not NULL. If it - * is NULL the value will be returned to "value_uint64" as int64. + * is NULL the value will be returned to "value_int64" as int64. * * Returns true if the value was found in the line. */ @@ -1108,3 +1478,14 @@ fileExists(const char *path) else return true; } + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/help.c b/src/help.c index d8726de3..1c9587ed 100644 --- a/src/help.c +++ b/src/help.c @@ -14,6 +14,7 @@ static void help_restore(void); static void help_validate(void); static void help_show(void); static void help_delete(void); +static void help_merge(void); static void help_set_config(void); static void help_show_config(void); static void help_add_instance(void); @@ -36,6 +37,8 @@ help_command(char *command) help_show(); else if (strcmp(command, "delete") == 0) help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); else if (strcmp(command, "set-config") == 0) help_set_config(); else if (strcmp(command, "show-config") == 0) @@ -113,7 +116,7 @@ help_pg_probackup(void) printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--replica-timeout=timeout]\n")); printf(_("\n %s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); @@ -136,6 +139,8 @@ help_pg_probackup(void) printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); printf(_("\n %s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); @@ -411,6 +416,17 @@ help_delete(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } +static void +help_merge(void) +{ + printf(_("%s merge -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); +} + static void help_set_config(void) { diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 00000000..9c754bbd --- /dev/null +++ b/src/merge.c @@ -0,0 +1,520 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + to_backup->wal_bytes = XLOG_SEG_SIZE * + (to_backup->stop_lsn / XLogSegSize - to_backup->start_lsn / XLogSegSize + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i; + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < parray_num(argument->files); i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index df143936..5e43c60c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -38,7 +38,7 @@ char backup_instance_path[MAXPGPATH]; char arclog_path[MAXPGPATH] = ""; /* common options */ -char *backup_id_string_param = NULL; +static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; bool progress = false; @@ -125,7 +125,7 @@ static pgut_option options[] = { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, - { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, /* backup options */ { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, @@ -206,7 +206,7 @@ main(int argc, char *argv[]) int rc; /* initialize configuration */ - pgBackup_init(¤t); + pgBackupInit(¤t); PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); @@ -235,10 +235,12 @@ main(int argc, char *argv[]) backup_subcmd = RESTORE_CMD; else if (strcmp(argv[1], "validate") == 0) backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW_CMD; else if (strcmp(argv[1], "delete") == 0) backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG_CMD; else if (strcmp(argv[1], "show-config") == 0) @@ -281,7 +283,8 @@ main(int argc, char *argv[]) if (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == VALIDATE_CMD || - backup_subcmd == DELETE_CMD) + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) { int i, len = 0, @@ -347,7 +350,8 @@ main(int argc, char *argv[]) } /* Option --instance is required for all commands except init and show */ - if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) { if (instance_name == NULL) elog(ERROR, "required parameter not specified: --instance"); @@ -359,7 +363,8 @@ main(int argc, char *argv[]) */ if (instance_name) { - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); /* @@ -402,18 +407,19 @@ main(int argc, char *argv[]) elog(ERROR, "-D, --pgdata must be an absolute path"); /* Sanity check of --backup-id option */ - if (backup_id_string_param != NULL) + if (backup_id_string != NULL) { - if (backup_subcmd != RESTORE_CMD - && backup_subcmd != VALIDATE_CMD - && backup_subcmd != DELETE_CMD - && backup_subcmd != SHOW_CMD) - elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", - argv[1]); + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); - current.backup_id = base36dec(backup_id_string_param); + current.backup_id = base36dec(backup_id_string); if (current.backup_id == 0) - elog(ERROR, "Invalid backup-id"); + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); } /* Setup stream options. They are used in streamutil.c. */ @@ -490,16 +496,19 @@ main(int argc, char *argv[]) case SHOW_CMD: return do_show(current.backup_id); case DELETE_CMD: - if (delete_expired && backup_id_string_param) + if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); - if (!delete_expired && !delete_wal && !backup_id_string_param) + if (!delete_expired && !delete_wal && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); - if (delete_wal && !delete_expired && !backup_id_string_param) + if (delete_wal && !delete_expired && !backup_id_string) return do_retention_purge(); if (delete_expired) return do_retention_purge(); else return do_delete(current.backup_id); + case MERGE_CMD: + do_merge(current.backup_id); + break; case SHOW_CONFIG_CMD: return do_configure(true); case SET_CONFIG_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 04683e65..f4c139e2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -122,8 +122,9 @@ typedef enum BackupStatus { BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ BACKUP_STATUS_OK, /* completed backup */ - BACKUP_STATUS_RUNNING, /* running backup */ BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ BACKUP_STATUS_DELETING, /* data files are being deleted */ BACKUP_STATUS_DELETED, /* data files have been deleted */ BACKUP_STATUS_DONE, /* completed but not validated yet */ @@ -144,15 +145,16 @@ typedef enum ProbackupSubcmd { NO_CMD = 0, INIT_CMD, - ARCHIVE_PUSH_CMD, - ARCHIVE_GET_CMD, ADD_INSTANCE_CMD, DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, BACKUP_CMD, RESTORE_CMD, VALIDATE_CMD, - SHOW_CMD, DELETE_CMD, + MERGE_CMD, + SHOW_CMD, SET_CONFIG_CMD, SHOW_CONFIG_CMD } ProbackupSubcmd; @@ -418,7 +420,8 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( bool target_immediate, const char *target_name, const char *target_action, bool restore_no_validate); -extern void opt_tablespace_map(pgut_option *opt, const char *arg); +/* in merge.c */ +extern void do_merge(time_t backup_id); /* in init.c */ extern int do_init(void); @@ -470,10 +473,15 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); @@ -481,10 +489,13 @@ extern int pgBackupCompareIdDesc(const void *f1, const void *f2); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root); -extern void list_data_directories(parray *files, const char *path, - bool is_root, bool exclude); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); extern parray *dir_read_file_list(const char *root, const char *file_txt); @@ -493,12 +504,13 @@ extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, bool omit_symlink); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(pgFile *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); @@ -506,13 +518,15 @@ extern int pgFileCompareSize(const void *f1, const void *f2); /* in data.c */ extern bool backup_data_file(backup_files_arg* arguments, - const char *from_root, const char *to_root, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - BackupMode backup_mode); -extern void restore_data_file(const char *from_root, const char *to_root, - pgFile *file, pgBackup *backup); -extern bool copy_file(const char *from_root, const char *to_root, - pgFile *file); + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); @@ -553,7 +567,6 @@ extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern pg_time_t timestamptz_to_time_t(TimestampTz t); extern int parse_server_version(char *server_version_str); -extern void pgBackup_init(pgBackup *backup); /* in status.c */ extern bool is_pg_running(void); diff --git a/src/restore.c b/src/restore.c index 2a0fec83..1464e4a0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -32,50 +32,12 @@ typedef struct int ret; } restore_files_arg; -/* Tablespace mapping structures */ - -typedef struct TablespaceListCell -{ - struct TablespaceListCell *next; - char old_dir[MAXPGPATH]; - char new_dir[MAXPGPATH]; -} TablespaceListCell; - -typedef struct TablespaceList -{ - TablespaceListCell *head; - TablespaceListCell *tail; -} TablespaceList; - -typedef struct TablespaceCreatedListCell -{ - struct TablespaceCreatedListCell *next; - char link_name[MAXPGPATH]; - char linked_dir[MAXPGPATH]; -} TablespaceCreatedListCell; - -typedef struct TablespaceCreatedList -{ - TablespaceCreatedListCell *head; - TablespaceCreatedListCell *tail; -} TablespaceCreatedList; - static void restore_backup(pgBackup *backup); -static void restore_directories(const char *pg_data_dir, - const char *backup_dir); -static void check_tablespace_mapping(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); -static const char *get_tablespace_mapping(const char *dir); -static void set_tablespace_created(const char *link, const char *dir); -static const char *get_tablespace_created(const char *link); - -/* Tablespace mapping */ -static TablespaceList tablespace_dirs = {NULL, NULL}; -static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* @@ -115,8 +77,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, catalog_lock(); /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backups == NULL) - elog(ERROR, "Failed to get backup list."); /* Find backup range we should restore or validate. */ for (i = 0; i < parray_num(backups); i++) @@ -256,6 +216,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, for (i = base_full_backup_index; i >= dest_backup_index; i--) { pgBackup *backup = (pgBackup *) parray_get(backups, i); + pgBackupValidate(backup); /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ if (backup->status == BACKUP_STATUS_CORRUPT) @@ -405,7 +366,7 @@ restore_backup(pgBackup *backup) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - restore_directories(pgdata, this_backup_path); + create_data_directories(pgdata, this_backup_path, true); /* * Get list of files which need to be restored. @@ -506,212 +467,6 @@ remove_deleted_files(pgBackup *backup) parray_free(files_restored); } -/* - * Restore backup directories from **backup_database_dir** to **pg_data_dir**. - * - * TODO: Think about simplification and clarity of the function. - */ -static void -restore_directories(const char *pg_data_dir, const char *backup_dir) -{ - parray *dirs, - *links; - size_t i; - char backup_database_dir[MAXPGPATH], - to_path[MAXPGPATH]; - - dirs = parray_new(); - links = parray_new(); - - join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); - - list_data_directories(dirs, backup_database_dir, true, false); - read_tablespace_map(links, backup_dir); - - elog(LOG, "restore directories and symlinks..."); - - for (i = 0; i < parray_num(dirs); i++) - { - pgFile *dir = (pgFile *) parray_get(dirs, i); - char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); - - Assert(S_ISDIR(dir->mode)); - - /* First try to create symlink and linked directory */ - if (path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) - { - char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), - *link_sep, - *tmp_ptr; - char link_name[MAXPGPATH]; - pgFile **link; - - /* Extract link name from relative path */ - link_sep = first_dir_separator(link_ptr); - if (link_sep != NULL) - { - int len = link_sep - link_ptr; - strncpy(link_name, link_ptr, len); - link_name[len] = '\0'; - } - else - goto create_directory; - - tmp_ptr = dir->path; - dir->path = link_name; - /* Search only by symlink name without path */ - link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); - dir->path = tmp_ptr; - - if (link) - { - const char *linked_path = get_tablespace_mapping((*link)->linked); - const char *dir_created; - - if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - linked_path); - - /* Check if linked directory was created earlier */ - dir_created = get_tablespace_created(link_name); - if (dir_created) - { - /* - * If symlink and linked directory were created do not - * create it second time. - */ - if (strcmp(dir_created, linked_path) == 0) - { - /* - * Create rest of directories. - * First check is there any directory name after - * separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - else - continue; - } - else - elog(ERROR, "tablespace directory \"%s\" of page backup does not " - "match with previous created tablespace directory \"%s\" of symlink \"%s\"", - linked_path, dir_created, link_name); - } - - if (link_sep) - elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", - linked_path, - (int) (link_sep - relative_ptr), relative_ptr); - else - elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", - linked_path, relative_ptr); - - /* Firstly, create linked directory */ - dir_create_dir(linked_path, DIR_PERMISSION); - - join_path_components(to_path, pg_data_dir, PG_TBLSPC_DIR); - /* Create pg_tblspc directory just in case */ - dir_create_dir(to_path, DIR_PERMISSION); - - /* Secondly, create link */ - join_path_components(to_path, to_path, link_name); - if (symlink(linked_path, to_path) < 0) - elog(ERROR, "could not create symbolic link \"%s\": %s", - to_path, strerror(errno)); - - /* Save linked directory */ - set_tablespace_created(link_name, linked_path); - - /* - * Create rest of directories. - * First check is there any directory name after separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - - continue; - } - } - -create_directory: - elog(LOG, "create directory \"%s\"", relative_ptr); - - /* This is not symlink, create directory */ - join_path_components(to_path, pg_data_dir, relative_ptr); - dir_create_dir(to_path, DIR_PERMISSION); - } - - parray_walk(links, pgFileFree); - parray_free(links); - - parray_walk(dirs, pgFileFree); - parray_free(dirs); -} - -/* - * Check that all tablespace mapping entries have correct linked directory - * paths. Linked directories must be empty or do not exist. - * - * If tablespace-mapping option is supplied, all OLDDIR entries must have - * entries in tablespace_map file. - */ -static void -check_tablespace_mapping(pgBackup *backup) -{ - char this_backup_path[MAXPGPATH]; - parray *links; - size_t i; - TablespaceListCell *cell; - pgFile *tmp_file = pgut_new(pgFile); - - links = parray_new(); - - pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - read_tablespace_map(links, this_backup_path); - - if (log_level_console <= LOG || log_level_file <= LOG) - elog(LOG, "check tablespace directories of backup %s", - base36enc(backup->start_time)); - - /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ - for (cell = tablespace_dirs.head; cell; cell = cell->next) - { - tmp_file->linked = cell->old_dir; - - if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) - elog(ERROR, "--tablespace-mapping option's old directory " - "doesn't have an entry in tablespace_map file: \"%s\"", - cell->old_dir); - } - - /* 2 - all linked directories must be empty */ - for (i = 0; i < parray_num(links); i++) - { - pgFile *link = (pgFile *) parray_get(links, i); - const char *linked_path = link->linked; - TablespaceListCell *cell; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - if (strcmp(link->linked, cell->old_dir) == 0) - { - linked_path = cell->new_dir; - break; - } - - if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - linked_path); - - if (!dir_is_empty(linked_path)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - } - - free(tmp_file); - parray_walk(links, pgFileFree); - parray_free(links); -} - /* * Restore files into $PGDATA. */ @@ -743,7 +498,6 @@ restore_files(void *arg) elog(LOG, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), rel_path); - /* * For PAGE and PTRACK backups skip files which haven't changed * since previous backup and thus were not backed up. @@ -754,11 +508,11 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(VERBOSE, "The file didn`t changed. Skip restore: %s", file->path); + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); continue; } - /* Directories was created before */ + /* Directories were created before */ if (S_ISDIR(file->mode)) { elog(VERBOSE, "directory, skip"); @@ -778,9 +532,18 @@ restore_files(void *arg) * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) - restore_data_file(from_root, pgdata, file, arguments->backup); + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } else copy_file(from_root, pgdata, file); @@ -1151,115 +914,3 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } - -/* - * Split argument into old_dir and new_dir and append to tablespace mapping - * list. - * - * Copy of function tablespace_list_append() from pg_basebackup.c. - */ -void -opt_tablespace_map(pgut_option *opt, const char *arg) -{ - TablespaceListCell *cell = pgut_new(TablespaceListCell); - char *dst; - char *dst_ptr; - const char *arg_ptr; - - dst_ptr = dst = cell->old_dir; - for (arg_ptr = arg; *arg_ptr; arg_ptr++) - { - if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "directory name too long"); - - if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') - ; /* skip backslash escaping = */ - else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) - { - if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); - else - dst = dst_ptr = cell->new_dir; - } - else - *dst_ptr++ = *arg_ptr; - } - - if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid tablespace mapping format \"%s\", " - "must be \"OLDDIR=NEWDIR\"", arg); - - /* - * This check isn't absolutely necessary. But all tablespaces are created - * with absolute directories, so specifying a non-absolute path here would - * just never match, possibly confusing users. It's also good to be - * consistent with the new_dir check. - */ - if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", - cell->old_dir); - - if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", - cell->new_dir); - - if (tablespace_dirs.tail) - tablespace_dirs.tail->next = cell; - else - tablespace_dirs.head = cell; - tablespace_dirs.tail = cell; -} - -/* - * Retrieve tablespace path, either relocated or original depending on whether - * -T was passed or not. - * - * Copy of function get_tablespace_mapping() from pg_basebackup.c. - */ -static const char * -get_tablespace_mapping(const char *dir) -{ - TablespaceListCell *cell; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - if (strcmp(dir, cell->old_dir) == 0) - return cell->new_dir; - - return dir; -} - -/* - * Save create directory path into memory. We can use it in next page restore to - * not raise the error "restore tablespace destination is not empty" in - * restore_directories(). - */ -static void -set_tablespace_created(const char *link, const char *dir) -{ - TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); - - strcpy(cell->link_name, link); - strcpy(cell->linked_dir, dir); - cell->next = NULL; - - if (tablespace_created_dirs.tail) - tablespace_created_dirs.tail->next = cell; - else - tablespace_created_dirs.head = cell; - tablespace_created_dirs.tail = cell; -} - -/* - * Is directory was created when symlink was created in restore_directories(). - */ -static const char * -get_tablespace_created(const char *link) -{ - TablespaceCreatedListCell *cell; - - for (cell = tablespace_created_dirs.head; cell; cell = cell->next) - if (strcmp(link, cell->link_name) == 0) - return cell->linked_dir; - - return NULL; -} diff --git a/src/show.c b/src/show.c index 01a558e0..f240ce93 100644 --- a/src/show.c +++ b/src/show.c @@ -247,8 +247,6 @@ show_instance(time_t requested_backup_id, bool show_name) parray *backup_list; backup_list = catalog_get_backup_list(requested_backup_id); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); if (show_format == SHOW_PLAIN) show_instance_plain(backup_list, show_name); diff --git a/src/util.c b/src/util.c index 798e3a1f..cb8f9bf6 100644 --- a/src/util.c +++ b/src/util.c @@ -271,8 +271,9 @@ status2str(BackupStatus status) { "UNKNOWN", "OK", - "RUNNING", "ERROR", + "RUNNING", + "MERGING", "DELETING", "DELETED", "DONE", @@ -322,36 +323,3 @@ remove_not_digit(char *buf, size_t len, const char *str) } buf[j] = '\0'; } - -/* Fill pgBackup struct with default values */ -void -pgBackup_init(pgBackup *backup) -{ - backup->backup_id = INVALID_BACKUP_ID; - backup->backup_mode = BACKUP_MODE_INVALID; - backup->status = BACKUP_STATUS_INVALID; - backup->tli = 0; - backup->start_lsn = 0; - backup->stop_lsn = 0; - backup->start_time = (time_t) 0; - backup->end_time = (time_t) 0; - backup->recovery_xid = 0; - backup->recovery_time = (time_t) 0; - - backup->data_bytes = BYTES_INVALID; - backup->wal_bytes = BYTES_INVALID; - - backup->compress_alg = COMPRESS_ALG_DEFAULT; - backup->compress_level = COMPRESS_LEVEL_DEFAULT; - - backup->block_size = BLCKSZ; - backup->wal_block_size = XLOG_BLCKSZ; - backup->checksum_version = 0; - - backup->stream = false; - backup->from_replica = false; - backup->parent_backup = 0; - backup->primary_conninfo = NULL; - backup->program_version[0] = '\0'; - backup->server_version[0] = '\0'; -} diff --git a/src/validate.c b/src/validate.c index 7130b1a5..7fa74cf1 100644 --- a/src/validate.c +++ b/src/validate.c @@ -195,7 +195,7 @@ pgBackupValidateFiles(void *arg) break; } - crc = pgFileGetCRC(file); + crc = pgFileGetCRC(file->path); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", @@ -290,8 +290,6 @@ do_validate_instance(void) /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backups == NULL) - elog(ERROR, "Failed to get backup list."); /* Valiate each backup along with its xlog files. */ for (i = 0; i < parray_num(backups); i++) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index d60d4d08..35f58406 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -50,7 +50,6 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] - pg_probackup restore -B backup-dir --instance=instance_name [-D pgdata-dir] [-i backup-id] [--progress] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] @@ -73,6 +72,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup delete -B backup-dir --instance=instance_name [--wal] [-i backup-id | --expired] + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + pg_probackup add-instance -B backup-dir -D pgdata-dir --instance=instance_name diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 3fd34de6..0d04d898 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -614,6 +614,16 @@ class ProbackupTest(object): return self.run_pb(cmd_list + options, async, gdb) + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + def restore_node( self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[] diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 00000000..1be3dd8b --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py index 34f7cd0f..8bd473fa 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -12,6 +12,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_help_1(self): """help options""" + self.maxDiff = None fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: From 92b7cc912d6fa425484a7342b5835325fbc4917a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 2 Aug 2018 13:48:29 +0300 Subject: [PATCH 93/99] PGPRO-1290: Fix a typo in check_server_version() --- src/backup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 09fe7cd6..eb9d4d96 100644 --- a/src/backup.c +++ b/src/backup.c @@ -989,23 +989,23 @@ check_server_version(void) if (PQresultStatus(res) == PGRES_FATAL_ERROR) /* It seems we connected to PostgreSQL (not Postgres Pro) */ elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection made with PostgreSQL %s", + "but connection is made with PostgreSQL %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection made with Postgres Pro %s %s", + "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str, PQgetvalue(res, 0, 0)); #else if (PQresultStatus(res) != PGRES_FATAL_ERROR) /* It seems we connected to Postgres Pro (not PostgreSQL) */ elog(ERROR, "%s was built with PostgreSQL %s, " - "but connection made with Postgres Pro %s %s", + "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, server_version_str, PQgetvalue(res, 0, 0)); else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) - elog(ERROR, "%s was built with PostgreSQL %s, but connection made with %s", + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", PROGRAM_NAME, PG_MAJORVERSION, server_version_str); #endif From 8f4232db349967d88bca8204997262f3e808a806 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 2 Aug 2018 17:35:48 +0300 Subject: [PATCH 94/99] add check --- src/restore.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 6cf1a6c3..661b4c45 100644 --- a/src/restore.c +++ b/src/restore.c @@ -203,9 +203,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup = current_backup; dest_backup_index = i; } - } + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + /* If we already found dest_backup, look for full backup. */ if (dest_backup) { @@ -238,6 +240,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } + if (base_full_backup == NULL) + elog(ERROR, "Full backup satisfying target options is not found."); + /* * Ensure that directories provided in tablespace mapping are valid * i.e. empty or not exist. From 9efc32f943dc4cd05407384d91200b3d3f094e26 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 3 Aug 2018 12:11:01 +0300 Subject: [PATCH 95/99] fix counting in dest_backup search --- src/catalog.c | 1 - src/restore.c | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index baf6dbae..6892db6d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -254,7 +254,6 @@ catalog_get_backup_list(time_t requested_backup_id) struct dirent *data_ent = NULL; parray *backups = NULL; pgBackup *backup = NULL; - int i; /* open backup instance backups directory */ diff --git a/src/restore.c b/src/restore.c index 661b4c45..dcc7e741 100644 --- a/src/restore.c +++ b/src/restore.c @@ -85,7 +85,7 @@ int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, bool is_restore) { - int i; + int i = 0; parray *backups; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; @@ -119,9 +119,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "Failed to get backup list."); /* Find backup range we should restore or validate. */ - for (i = 0; i < parray_num(backups); i++) + while ((i < parray_num(backups)) && !dest_backup) { current_backup = (pgBackup *) parray_get(backups, i); + i++; /* Skip all backups which started after target backup */ if (target_backup_id && current_backup->start_time > target_backup_id) @@ -133,7 +134,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (is_restore && - !dest_backup && target_backup_id == INVALID_BACKUP_ID && current_backup->status != BACKUP_STATUS_OK) { @@ -147,8 +147,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * ensure that it satisfies recovery target. */ if ((target_backup_id == current_backup->start_time - || target_backup_id == INVALID_BACKUP_ID) - && !dest_backup) + || target_backup_id == INVALID_BACKUP_ID)) { /* backup is not ok, @@ -201,7 +200,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Save it as dest_backup */ dest_backup = current_backup; - dest_backup_index = i; + dest_backup_index = i-1; } } From 68060e7dda171dd77c72d1dbdc0f250ca58904b7 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 30 Jul 2018 19:22:08 +0300 Subject: [PATCH 96/99] fix race in make_pagemap_from_ptrack() for segments created after we've read ptrack --- src/backup.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index eb9d4d96..9a4c7005 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2407,19 +2407,32 @@ make_pagemap_from_ptrack(parray *files) */ start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; - if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) { - file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; } else { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); - memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } } else { From 2fac44e18e1d8c8e696984af4308246947f6a068 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 3 Aug 2018 13:29:07 +0300 Subject: [PATCH 97/99] fix merge conflits --- src/catalog.c | 1 + src/util.c | 34 ---------------------------------- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index a277f605..51d791a7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -783,6 +783,7 @@ pgBackupInit(pgBackup *backup) backup->stream = false; backup->from_replica = false; backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; backup->primary_conninfo = NULL; backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; diff --git a/src/util.c b/src/util.c index 1e837745..cb8f9bf6 100644 --- a/src/util.c +++ b/src/util.c @@ -323,37 +323,3 @@ remove_not_digit(char *buf, size_t len, const char *str) } buf[j] = '\0'; } - -/* Fill pgBackup struct with default values */ -void -pgBackup_init(pgBackup *backup) -{ - backup->backup_id = INVALID_BACKUP_ID; - backup->backup_mode = BACKUP_MODE_INVALID; - backup->status = BACKUP_STATUS_INVALID; - backup->tli = 0; - backup->start_lsn = 0; - backup->stop_lsn = 0; - backup->start_time = (time_t) 0; - backup->end_time = (time_t) 0; - backup->recovery_xid = 0; - backup->recovery_time = (time_t) 0; - - backup->data_bytes = BYTES_INVALID; - backup->wal_bytes = BYTES_INVALID; - - backup->compress_alg = COMPRESS_ALG_DEFAULT; - backup->compress_level = COMPRESS_LEVEL_DEFAULT; - - backup->block_size = BLCKSZ; - backup->wal_block_size = XLOG_BLCKSZ; - backup->checksum_version = 0; - - backup->stream = false; - backup->from_replica = false; - backup->parent_backup = 0; - backup->parent_backup_link = NULL; - backup->primary_conninfo = NULL; - backup->program_version[0] = '\0'; - backup->server_version[0] = '\0'; -} From f508308a74d1dacc5839ac5b2f1b610423e8c41b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 3 Aug 2018 13:58:16 +0300 Subject: [PATCH 98/99] PGPRO-1449: Add logging options in help --- src/help.c | 33 ++++++++++++++++++++++++++++++++- src/merge.c | 9 +++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/help.c b/src/help.c index 1c9587ed..9ad6cc2e 100644 --- a/src/help.c +++ b/src/help.c @@ -420,11 +420,42 @@ static void help_merge(void) { printf(_("%s merge -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id\n\n")); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceed this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'KB', 'MB', 'GB', 'TB' (default: KB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceed this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } static void diff --git a/src/merge.c b/src/merge.c index 9c754bbd..b149ef5b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -353,14 +353,15 @@ merge_files(void *arg) pgBackup *to_backup = argument->to_backup; pgBackup *from_backup = argument->from_backup; char tmp_file_path[MAXPGPATH]; - int i; + int i, + num_files = parray_num(argument->files); int to_root_len = strlen(argument->to_root); if (to_backup->compress_alg == PGLZ_COMPRESS || to_backup->compress_alg == ZLIB_COMPRESS) join_path_components(tmp_file_path, argument->to_root, "tmp"); - for (i = 0; i < parray_num(argument->files); i++) + for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); @@ -371,6 +372,10 @@ merge_files(void *arg) if (interrupted) elog(ERROR, "Interrupted during merging backups"); + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + /* * Skip files which haven't changed since previous backup. But in case * of DELTA backup we should consider n_blocks to truncate the target From 3f5540de6a9d9107b712f4eaef2baf436457c213 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 6 Aug 2018 12:47:51 +0300 Subject: [PATCH 99/99] Version 2.0.18 --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5e43c60c..a18ee5c3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -19,7 +19,7 @@ #include #include "pg_getopt.h" -const char *PROGRAM_VERSION = "2.0.17"; +const char *PROGRAM_VERSION = "2.0.18"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index a1257673..35e212c3 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.17 \ No newline at end of file +pg_probackup 2.0.18 \ No newline at end of file