From 2689ddb9ba9c5bf9af4157beadf7d4f05a5ef855 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 26 Feb 2018 17:55:39 +0300 Subject: [PATCH 01/66] use dynamically allocated array for backup_threads --- src/backup.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 08cc990d..93924dbc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -438,8 +438,9 @@ do_backup_instance(void) char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - pthread_t backup_threads[num_threads]; - backup_files_args *backup_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *backup_threads; + backup_files_args *backup_threads_args; pgBackup *prev_backup = NULL; char prev_backup_filelist_path[MAXPGPATH]; @@ -639,9 +640,12 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileCompareSize); /* init thread args with own file lists */ + backup_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + backup_threads_args = (backup_files_args *) palloc(sizeof(backup_files_args)*num_threads); + for (i = 0; i < num_threads; i++) { - backup_files_args *arg = pg_malloc(sizeof(backup_files_args)); + backup_files_args *arg = &(backup_threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; @@ -650,30 +654,29 @@ do_backup_instance(void) arg->prev_backup_start_lsn = prev_backup_start_lsn; arg->thread_backup_conn = NULL; arg->thread_cancel_conn = NULL; - backup_threads_args[i] = arg; } /* Run threads */ elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { + backup_files_args *arg = &(backup_threads_args[i]); elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, - backup_threads_args[i]); + arg); else pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) remote_backup_files, - backup_threads_args[i]); + arg); } /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); - pg_free(backup_threads_args[i]); } elog(LOG, "Data files are transfered"); From 1adde8f19f5e6abf2103a8e5a9bb1e9ab7caf8eb Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 26 Feb 2018 18:44:22 +0300 Subject: [PATCH 02/66] fix usage of symlink specific code --- src/backup.c | 4 ++++ src/dir.c | 16 +++++++++++++++- src/restore.c | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 93924dbc..6e636308 100644 --- a/src/backup.c +++ b/src/backup.c @@ -239,7 +239,11 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) else if (copybuf[156] == '2') { /* Symlink */ +#ifndef WIN32 pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif } else elog(ERROR, "Unrecognized link indicator \"%c\"\n", diff --git a/src/dir.c b/src/dir.c index 77b37274..eed62c5c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -380,7 +380,13 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, * Add to files list only files, links and directories. Skip sockets and * other unexpected file formats. */ - if (!S_ISDIR(file->mode) && !S_ISLNK(file->mode) && !S_ISREG(file->mode)) + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode) && +#ifndef WIN32 + !(S_ISLNK(file->mode)) +#else + !(pgwin32_is_junction(file->path)) +#endif + ) { elog(WARNING, "Skip file \"%s\": unexpected file format", file->path); return; @@ -430,7 +436,11 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, } /* chase symbolic link chain and find regular file or directory */ + #ifndef WIN32 while (S_ISLNK(file->mode)) + #else + while (pgwin32_is_junction(file->path)) + #endif { ssize_t len; char linked[MAXPGPATH]; @@ -718,7 +728,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); +#ifndef WIN32 if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif fprintf(out, ",\"linked\":\"%s\"", file->linked); fprintf(out, "}\n"); diff --git a/src/restore.c b/src/restore.c index 5965fe19..fb853b6a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -85,7 +85,7 @@ do_restore_or_validate(time_t target_backup_id, { int i; parray *backups; - parray *timelines; + parray *timelines = NULL; pgRecoveryTarget *rt = NULL; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; From 2a9baaa960db6782596be68011bdb80284548efe Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 12:31:41 +0300 Subject: [PATCH 03/66] Rename 'delete' in the ProbackupSubcmd enum to differ it from reserved keyword --- src/pg_probackup.c | 8 ++++---- src/pg_probackup.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 88270042..6fc55840 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -210,7 +210,7 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "show") == 0) backup_subcmd = SHOW; else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; + backup_subcmd = DELETE_SUBCMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG; else if (strcmp(argv[1], "show-config") == 0) @@ -256,7 +256,7 @@ main(int argc, char *argv[]) if (backup_subcmd == BACKUP || backup_subcmd == RESTORE || backup_subcmd == VALIDATE || - backup_subcmd == DELETE) + backup_subcmd == DELETE_SUBCMD) { int i, len = 0, @@ -377,7 +377,7 @@ main(int argc, char *argv[]) { if (backup_subcmd != RESTORE && backup_subcmd != VALIDATE - && backup_subcmd != DELETE + && backup_subcmd != DELETE_SUBCMD && backup_subcmd != SHOW) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -456,7 +456,7 @@ main(int argc, char *argv[]) false); case SHOW: return do_show(current.backup_id); - case DELETE: + case DELETE_SUBCMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3f34e8f9..1c1eef1b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -141,7 +141,7 @@ typedef enum ProbackupSubcmd RESTORE, VALIDATE, SHOW, - DELETE, + DELETE_SUBCMD, SET_CONFIG, SHOW_CONFIG } ProbackupSubcmd; From 18364caa34c71263fb575bc07012620274e66803 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 13:02:43 +0300 Subject: [PATCH 04/66] rewrite dir_create_dir() in a cross platform way --- src/dir.c | 8 +++++--- src/pg_probackup.h | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index eed62c5c..834b2708 100644 --- a/src/dir.c +++ b/src/dir.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include #include #include @@ -99,10 +98,12 @@ int dir_create_dir(const char *dir, mode_t mode) { char copy[MAXPGPATH]; - char parent[MAXPGPATH]; + char *parent; strncpy(copy, dir, MAXPGPATH); - strncpy(parent, dirname(copy), MAXPGPATH); + + parent = pstrdup(dir); + get_parent_directory(parent); /* Create parent first */ if (access(parent, F_OK) == -1) @@ -116,6 +117,7 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } + pfree(parent); return 0; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1c1eef1b..392ef579 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -28,6 +28,7 @@ #include "storage/checksum.h" #include "utils/pg_crc.h" #include "common/relpath.h" +#include "port.h" #include "utils/parray.h" #include "utils/pgut.h" From 7e096be2b3b440e92a63ef29d77a57ceeb4f64cc Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 14:37:46 +0300 Subject: [PATCH 05/66] add partial ptread implementation for windows port --- src/backup.c | 104 ++++++++++++++++++++++++++++++++++++++++++++- src/pg_probackup.h | 4 ++ src/restore.c | 6 ++- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6e636308..0ee2a9ba 100644 --- a/src/backup.c +++ b/src/backup.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "libpq/pqsignal.h" #include "storage/bufpage.h" @@ -29,6 +28,16 @@ #include "streamutil.h" #include "pgtar.h" +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; @@ -2763,3 +2772,96 @@ pg_ptrack_get_block(backup_files_args *arguments, return result; } + +#ifdef WIN32 /* WIN32 */ +volatile bool timer_exceeded = false; /* flag from signal handler */ + +static VOID CALLBACK +win32_timer_callback(PVOID lpParameter, BOOLEAN TimerOrWaitFired) +{ timer_exceeded = true; } + +static void +setalarm(int seconds) +{ + HANDLE queue; + HANDLE timer; + + /* This function will be called at most once, so we can cheat a bit. */ + queue = CreateTimerQueue(); + if (seconds > ((DWORD)-1) / 1000 || + !CreateTimerQueueTimer(&timer, queue, + win32_timer_callback, NULL, seconds * 1000, 0, + WT_EXECUTEINTIMERTHREAD | WT_EXECUTEONLYONCE)) + { + fprintf(stderr, "failed to set timer\n"); exit(1); + } +} + +/* partial pthread implementation for Windows */ + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} + +win32_pthread; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + th->result = th->routine(th->arg); + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, + win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 392ef579..7da50800 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -17,6 +17,9 @@ #ifndef WIN32 #include +#include +#else +#include "port/pthread-win32.h" #endif #include "access/timeline.h" @@ -30,6 +33,7 @@ #include "common/relpath.h" #include "port.h" + #include "utils/parray.h" #include "utils/pgut.h" diff --git a/src/restore.c b/src/restore.c index fb853b6a..c983ed0b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -14,10 +14,14 @@ #include #include #include -#include #include "catalog/pg_control.h" +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#endif + typedef struct { parray *files; From 834cfb6ab82c027dfd3ba8975d28b25fb5f7c066 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 27 Feb 2018 14:40:05 +0300 Subject: [PATCH 06/66] fix 'unary minus operator applied to unsigned type' --- src/restore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index c983ed0b..a63a4bd3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -913,7 +913,8 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - entry->end = (uint32) (-1UL << 32) | -1UL; + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; parray_insert(result, 0, entry); return result; From 9ec02abc0df1843d81201a68c135407de4c55361 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 2 Mar 2018 15:22:57 +0300 Subject: [PATCH 07/66] WIP. Doesn't work. Cross platform multi threaded backup/restore --- Makefile | 7 +++++-- src/backup.c | 6 +++--- src/pg_probackup.h | 3 ++- src/restore.c | 4 ++-- src/validate.c | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 0f4fa672..e7106d55 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,10 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ - src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h \ + src/atomics.h -INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h src/atomics.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -61,6 +62,8 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ +src/atomics.h: $(top_srcdir)/src/include/port/atomics.h + rm -f $@ && $(LN_S) $(srchome)/src/include/port/atomics.h $@ ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c diff --git a/src/backup.c b/src/backup.c index 0ee2a9ba..77ca3311 100644 --- a/src/backup.c +++ b/src/backup.c @@ -383,7 +383,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); @@ -646,7 +646,7 @@ do_backup_instance(void) } /* setup threads */ - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* sort by size for load balancing */ @@ -1922,7 +1922,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7da50800..713b8e62 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -33,6 +33,7 @@ #include "common/relpath.h" #include "port.h" +#include "atomics.h" #include "utils/parray.h" #include "utils/pgut.h" @@ -105,7 +106,7 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile uint32 lock; /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; diff --git a/src/restore.c b/src/restore.c index a63a4bd3..24f6bfae 100644 --- a/src/restore.c +++ b/src/restore.c @@ -398,7 +398,7 @@ restore_backup(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ @@ -703,7 +703,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, diff --git a/src/validate.c b/src/validate.c index 2be75760..e6bb3b22 100644 --- a/src/validate.c +++ b/src/validate.c @@ -62,7 +62,7 @@ pgBackupValidate(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Validate files */ @@ -116,7 +116,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) From dff728e2720e5818bd8af4275378ac0fc998d81e Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Mar 2018 13:12:54 +0300 Subject: [PATCH 08/66] Add scripts for generate Visual Studio Project --- msvs/pg_probackup.sln | 28 ++++ msvs/template.pg_probackup.vcxproj | 203 ++++++++++++++++++++++ msvs/template.pg_probackup_2.vcxproj | 203 ++++++++++++++++++++++ win32build.pl | 240 +++++++++++++++++++++++++++ win32build_2.pl | 219 ++++++++++++++++++++++++ 5 files changed, 893 insertions(+) create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 win32build.pl create mode 100644 win32build_2.pl diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..3c616170 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..66c71b8b --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From 0c9bef39094eee3da418a73e09c3f827633d30a3 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 13:31:36 +0300 Subject: [PATCH 09/66] update VS project --- msvs/template.pg_probackup.vcxproj | 8 ++++---- msvs/template.pg_probackup_2.vcxproj | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 3c616170..8c731e28 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -68,22 +68,22 @@ true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj index 66c71b8b..2fc101a4 100644 --- a/msvs/template.pg_probackup_2.vcxproj +++ b/msvs/template.pg_probackup_2.vcxproj @@ -68,22 +68,22 @@ true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../src;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) From b7457d619fcad9f7696e409da1caa4f4f5f0816f Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 13:33:25 +0300 Subject: [PATCH 10/66] update VS project --- doit.cmd | 1 + 1 file changed, 1 insertion(+) create mode 100644 doit.cmd diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..cc29bc88 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file From b637a98b50e47ecee43a3e9715dc052f2a7df12d Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 14:53:04 +0300 Subject: [PATCH 11/66] include postgresql cross platform atomics. It takes ugly #undef FRONTEND hack to do that --- Makefile | 8 +++----- src/pg_probackup.h | 6 +++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e7106d55..f533d814 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,9 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ - src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h \ - src/atomics.h + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h -INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h src/atomics.h +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -62,8 +61,7 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -src/atomics.h: $(top_srcdir)/src/include/port/atomics.h - rm -f $@ && $(LN_S) $(srchome)/src/include/port/atomics.h $@ + ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 713b8e62..472b1d66 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -33,7 +33,11 @@ #include "common/relpath.h" #include "port.h" -#include "atomics.h" +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif #include "utils/parray.h" #include "utils/pgut.h" From f0547f46a158aa7eff415a2c495286a87468eea4 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 15:17:58 +0300 Subject: [PATCH 12/66] fix includes --- src/catalog.c | 1 - src/utils/logger.c | 8 +++++++- src/validate.c | 8 +++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 0e9b4de2..e82c434a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/src/utils/logger.c b/src/utils/logger.c index 5d51394f..3a9542d6 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -17,6 +16,13 @@ #include "logger.h" #include "pgut.h" +#ifndef WIN32 +#include +#include +#else +#include "port/pthread-win32.h" +#endif + /* Logger parameters */ int log_level_console = LOG_NONE; diff --git a/src/validate.c b/src/validate.c index e6bb3b22..e9f3bb9f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -11,9 +11,15 @@ #include "pg_probackup.h" #include -#include #include +#ifndef WIN32 +#include +#include +#else +#include "port/pthread-win32.h" +#endif + static void pgBackupValidateFiles(void *arg); static void do_validate_instance(void); From 3f91c372a5c377c89335b3994b28840b436540e1 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 15:18:15 +0300 Subject: [PATCH 13/66] update VS project --- msvs/template.pg_probackup.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 8c731e28..03caf86c 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -68,22 +68,22 @@ true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) @PGROOT@\lib;@$(LibraryPath) From 990ccd32fc631178bb772d69f123e7dd861da870 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 15:24:49 +0300 Subject: [PATCH 14/66] use dynamically allocated array for restore_threads --- src/restore.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/restore.c b/src/restore.c index 24f6bfae..d8fd4221 100644 --- a/src/restore.c +++ b/src/restore.c @@ -349,8 +349,10 @@ restore_backup(pgBackup *backup) char list_path[MAXPGPATH]; parray *files; int i; - pthread_t restore_threads[num_threads]; - restore_files_args *restore_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *restore_threads; + restore_files_args *restore_threads_args; + if (backup->status != BACKUP_STATUS_OK) elog(ERROR, "Backup %s cannot be restored because it is not valid", @@ -394,6 +396,9 @@ restore_backup(pgBackup *backup) pgFileFree(parray_remove(files, i)); } + restore_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + restore_threads_args = (restore_files_args *) palloc(sizeof(restore_files_args)*num_threads); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -404,13 +409,13 @@ restore_backup(pgBackup *backup) /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = pg_malloc(sizeof(restore_files_args)); + restore_files_args *arg = &(restore_threads_args[i]); + arg->files = files; arg->backup = backup; elog(LOG, "Start thread for num:%li", parray_num(files)); - restore_threads_args[i] = arg; pthread_create(&restore_threads[i], NULL, (void *(*)(void *)) restore_files, arg); } @@ -418,9 +423,11 @@ restore_backup(pgBackup *backup) for (i = 0; i < num_threads; i++) { pthread_join(restore_threads[i], NULL); - pg_free(restore_threads_args[i]); } + pfree(restore_threads); + pfree(restore_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 14fa9542d43af91479ec8a75c86b8978ad8a6f03 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 17:02:58 +0300 Subject: [PATCH 15/66] use dynamically allocated array for validate_threads --- src/validate.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/validate.c b/src/validate.c index e9f3bb9f..96e06748 100644 --- a/src/validate.c +++ b/src/validate.c @@ -20,6 +20,16 @@ #include "port/pthread-win32.h" #endif +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + static void pgBackupValidateFiles(void *arg); static void do_validate_instance(void); @@ -41,8 +51,9 @@ pgBackupValidate(pgBackup *backup) char path[MAXPGPATH]; parray *files; bool corrupted = false; - pthread_t validate_threads[num_threads]; - validate_files_args *validate_threads_args[num_threads]; + /* arrays with meta info for multi threaded validate */ + pthread_t *validate_threads; + validate_files_args *validate_threads_args; int i; if (backup->status != BACKUP_STATUS_OK && @@ -71,25 +82,31 @@ pgBackupValidate(pgBackup *backup) pg_atomic_clear_flag(&file->lock); } + /* init thread args with own file lists */ + validate_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + validate_threads_args = (validate_files_args *) palloc(sizeof(validate_files_args)*num_threads); + /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = pg_malloc(sizeof(validate_files_args)); + validate_files_args *arg = &(validate_threads_args[i]); arg->files = files; arg->corrupted = false; - validate_threads_args[i] = arg; pthread_create(&validate_threads[i], NULL, (void *(*)(void *)) pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { + validate_files_args *arg = &(validate_threads_args[i]); pthread_join(validate_threads[i], NULL); - if (validate_threads_args[i]->corrupted) + if (arg->corrupted) corrupted = true; - pg_free(validate_threads_args[i]); } + pfree(validate_threads); + pfree(validate_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 351b3dd263e04c2e965ba9732c4abe63c0fba9ab Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 13 Mar 2018 17:41:03 +0300 Subject: [PATCH 16/66] define pthread_t in logger.c for WINDOWS --- src/utils/logger.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/logger.c b/src/utils/logger.c index 3a9542d6..a380888e 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -23,6 +23,16 @@ #include "port/pthread-win32.h" #endif +#ifdef WIN32 +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +int pthread_join(pthread_t th, void **thread_return); +#endif + /* Logger parameters */ int log_level_console = LOG_NONE; From dbd757761c1da90ef8bb52b114c5810b582396a8 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 13 Mar 2018 18:45:52 +0300 Subject: [PATCH 17/66] update VS project --- msvs/template.pg_probackup.vcxproj | 14 ++++++++++---- src/data.c | 2 +- src/utils/logger.c | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 03caf86c..044a3e73 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -69,22 +69,26 @@ true ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + true ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + false ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + false ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) + @PGROOT@\lib;$(LibraryPath) + @@ -134,6 +138,7 @@ true true @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) libc;%(IgnoreSpecificDefaultLibraries) @@ -154,6 +159,7 @@ true true @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) libc;%(IgnoreSpecificDefaultLibraries) diff --git a/src/data.c b/src/data.c index 95966ace..6a76a844 100644 --- a/src/data.c +++ b/src/data.c @@ -853,7 +853,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out; + FILE *out=NULL; char buf[XLOG_BLCKSZ]; const char *to_path_p = to_path; char to_path_temp[MAXPGPATH]; diff --git a/src/utils/logger.c b/src/utils/logger.c index a380888e..5a069fa3 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -143,8 +143,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) bool write_to_file, write_to_error_log, write_to_stderr; - va_list error_args, - std_args; + va_list error_args=NULL, + std_args=NULL; time_t log_time = (time_t) time(NULL); char strfbuf[128]; From c7bfcdbcd359fe78f03a502f5fddcd47f0f20955 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Thu, 15 Mar 2018 17:49:08 +0300 Subject: [PATCH 18/66] changed char to unsigned char for sname in the struct pgut_option --- src/utils/pgut.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/pgut.h b/src/utils/pgut.h index def0f9d2..d956e138 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -58,7 +58,7 @@ typedef enum pgut_optsrc typedef struct pgut_option { char type; - char sname; /* short name */ + unsigned char sname; /* short name */ const char *lname; /* long name */ void *var; /* pointer to variable */ pgut_optsrc allowed; /* allowed source */ From e54b151df0bcf118e32da4552c5c5812864ad65c Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Sat, 17 Mar 2018 19:04:20 +0300 Subject: [PATCH 19/66] changes in the parsing command line options --- src/utils/pgut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6b5d9b6e..515f35c8 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) + if (opts[i].val > 128 || !isprint(opts[i].val)) continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1053,7 +1053,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + while ((c = getopt_long(argc>2 ? argc - 1 : argc, argc>2 ? argv + 1: argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); if (opt && opt->allowed < SOURCE_CMDLINE) From 84358f6963de6bb8f8766ac85ba7e9c9c1877c53 Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Sat, 17 Mar 2018 19:22:26 +0300 Subject: [PATCH 20/66] changes in the parsing command line options --- src/utils/pgut.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 515f35c8..1573da0f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1046,6 +1046,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; + len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1053,7 +1054,8 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - while ((c = getopt_long(argc>2 ? argc - 1 : argc, argc>2 ? argv + 1: argv, optstring, longopts, &optindex)) != -1) + optind = 2; + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); if (opt && opt->allowed < SOURCE_CMDLINE) From d0092bb3c951816087a73f1b396c3c12789495fa Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Tue, 20 Mar 2018 11:15:53 +0300 Subject: [PATCH 21/66] changes in the parsing command line options --- src/utils/pgut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 1573da0f..08c54b11 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (opts[i].val > 128 || !isprint(opts[i].val)) + if (!isprint(opts[i].val)) //opts[i].val > 128 || continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) From 7e8cffb23fb86bff2cb336e66796bcade1147faa Mon Sep 17 00:00:00 2001 From: VictorSpirin Date: Thu, 22 Mar 2018 18:22:23 +0300 Subject: [PATCH 22/66] some changes --- src/backup.c | 17 +++++++++++++---- src/configure.c | 3 ++- src/dir.c | 4 +++- src/pg_probackup.c | 1 + src/pg_probackup.h | 10 ++++++++++ src/utils/pgut.c | 2 +- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 77ca3311..5c792933 100644 --- a/src/backup.c +++ b/src/backup.c @@ -552,9 +552,16 @@ do_backup_instance(void) join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); dir_create_dir(dst_backup_path, DIR_PERMISSION); + if (start_stream_mut == NULL){ + pthread_mutex_init(&start_stream_mut,NULL); + } + pthread_mutex_lock(&start_stream_mut); pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); pthread_mutex_lock(&start_stream_mut); +#ifdef WIN32 + sleep(10); +#endif if (conn == NULL) elog(ERROR, "Cannot continue backup because stream connect has failed."); @@ -936,7 +943,7 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - + if (system_id_conn != system_identifier) elog(ERROR, "Backup data directory was initialized for system id %ld, but connected instance system id is %ld", system_identifier, system_id_conn); @@ -959,14 +966,15 @@ confirm_block_size(const char *name, int blcksz) res = pgut_execute(backup_conn, "SELECT current_setting($1)", 1, &name, true); if (PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); - + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - PQclear(res); - if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); + + PQclear(res);//bad pointer to endp + } /* @@ -2864,4 +2872,5 @@ pthread_join(pthread_t th, void **thread_return) return 0; } + #endif /* WIN32 */ diff --git a/src/configure.c b/src/configure.c index 55d2bbc5..fa165190 100644 --- a/src/configure.c +++ b/src/configure.c @@ -114,7 +114,8 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); + //fprintf(out, "system-identifier = %li\n", config->system_identifier); + fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/dir.c b/src/dir.c index 834b2708..2069964a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -150,6 +150,8 @@ pgFileInit(const char *path) pgFile *file; file = (pgFile *) pgut_malloc(sizeof(pgFile)); + file->name = 0; + file->size = 0; file->mode = 0; file->read_size = 0; @@ -520,7 +522,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude, break; } } - else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + else if (file->name!=0 && strcmp(file->name, pgdata_exclude_dir[i]) == 0) { skip = true; break; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6fc55840..0206da22 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -434,6 +434,7 @@ main(int argc, char *argv[]) start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 472b1d66..cb84d4b4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -506,5 +506,15 @@ extern void pgBackup_init(pgBackup *backup); /* in status.c */ extern bool is_pg_running(void); +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#endif +#endif #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 08c54b11..de006657 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1221,7 +1221,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "# \n\r\t\v"); + i = j = strcspn(s, "#\n\r\t\v");//removed space memcpy(dst, s, j); } From e7665c1d504edca2b7784650af5490957fb2ca3d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 10 Apr 2018 18:44:51 +0300 Subject: [PATCH 23/66] Arthur Zakirov: fopen() in binary mode --- src/data.c | 8 ++++---- src/pg_probackup.h | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index 6a76a844..6934e627 100644 --- a/src/data.c +++ b/src/data.c @@ -865,7 +865,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, "rb"); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -896,7 +896,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, "wb"); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1018,7 +1018,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, "rb"); if (in == NULL) { #ifdef HAVE_LIBZ @@ -1055,7 +1055,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, "wb"); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index cb84d4b4..9a3d8e8c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -514,6 +514,7 @@ extern bool is_pg_running(void); #define read _read #define close _close #define write _write +#define mkdir(dir,mode) _mkdir(dir) #endif #endif From 03412d766f6b754413412668dfcf5d2694874413 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 24 Apr 2018 12:59:25 +0300 Subject: [PATCH 24/66] Add build script and project template for the 9.6 version --- doit96.cmd | 1 + msvs/template.pg_probackup96.vcxproj | 207 +++++++++++++++++++++++ win32build96.pl | 240 +++++++++++++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 doit96.cmd create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 win32build96.pl diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..c095d7cc --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,207 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + From 9a1e5ba0445384c9e531b208a42d304b2ef861b4 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 26 Apr 2018 15:02:03 +0300 Subject: [PATCH 25/66] Function pg_atomic_test_set_flag returns true if the flag has successfully been set, false otherwise --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5c792933..05a5c6df 100644 --- a/src/backup.c +++ b/src/backup.c @@ -692,7 +692,7 @@ do_backup_instance(void) (void *(*)(void *)) remote_backup_files, arg); } - + /* Wait threads */ for (i = 0; i < num_threads; i++) { @@ -1930,7 +1930,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ From 83d64c5b9d0d2ee4c7cc5e9a128108799bb279de Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 26 Apr 2018 15:23:50 +0300 Subject: [PATCH 26/66] Function pg_atomic_test_set_flag returns true if the flag has successfully been set, false otherwise --- src/backup.c | 2 +- src/restore.c | 2 +- src/validate.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 05a5c6df..0c65f728 100644 --- a/src/backup.c +++ b/src/backup.c @@ -383,7 +383,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); diff --git a/src/restore.c b/src/restore.c index d8fd4221..d91678cb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -710,7 +710,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, diff --git a/src/validate.c b/src/validate.c index 96e06748..39c40bdd 100644 --- a/src/validate.c +++ b/src/validate.c @@ -139,7 +139,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) From ac98fefc6e6b0d45df28efe2f6eefa1c39bf4510 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Fri, 11 May 2018 18:17:54 +0300 Subject: [PATCH 27/66] change mode for fopen w->PG_BINARY_W, r->PG_BINARY_R --- src/backup.c | 7 ++++--- src/data.c | 14 +++++++------- src/dir.c | 4 ++-- src/pg_probackup.c | 2 +- src/status.c | 2 +- src/utils/logger.c | 4 ++-- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0c65f728..f39a79f3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -287,7 +287,7 @@ static void remote_copy_file(PGconn *conn, pgFile* file) DATABASE_DIR); join_path_components(to_path, database_path, file->path); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -562,6 +562,7 @@ do_backup_instance(void) #ifdef WIN32 sleep(10); #endif + if (conn == NULL) elog(ERROR, "Cannot continue backup because stream connect has failed."); @@ -1715,7 +1716,7 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, "w"); + fp = fopen(backup_label, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1763,7 +1764,7 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, "w"); + fp = fopen(tablespace_map, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); diff --git a/src/data.c b/src/data.c index 6934e627..0d4ba525 100644 --- a/src/data.c +++ b/src/data.c @@ -421,7 +421,7 @@ backup_data_file(backup_files_args* arguments, INIT_CRC32C(file->crc); /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(file->crc); @@ -455,7 +455,7 @@ backup_data_file(backup_files_args* arguments, /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -547,7 +547,7 @@ restore_data_file(const char *from_root, BlockNumber blknum; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { elog(ERROR, "cannot open backup file \"%s\": %s", file->path, @@ -562,7 +562,7 @@ restore_data_file(const char *from_root, join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, "r+"); if (out == NULL && errno == ENOENT) - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -694,7 +694,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); @@ -710,7 +710,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -1189,7 +1189,7 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); diff --git a/src/dir.c b/src/dir.c index 2069964a..cf825542 100644 --- a/src/dir.c +++ b/src/dir.c @@ -216,7 +216,7 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, "r"); + fp = fopen(file->path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", file->path, strerror(errno)); @@ -333,7 +333,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, "r"); + black_list_file = fopen(path, PG_BINARY_R); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0206da22..de4394a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -181,13 +181,13 @@ main(int argc, char *argv[]) /* Check if backup_path is directory. */ struct stat stat_buf; int rc; - /* initialize configuration */ pgBackup_init(¤t); PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); + /* Parse subcommands and non-subcommand options */ if (argc > 1) { diff --git a/src/status.c b/src/status.c index 1c7c6038..155a07f4 100644 --- a/src/status.c +++ b/src/status.c @@ -38,7 +38,7 @@ get_pgpid(void) snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); - pidf = fopen(pid_file, "r"); + pidf = fopen(pid_file, PG_BINARY_R); if (pidf == NULL) { /* No pid file, not an error on startup */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 5a069fa3..09cad75d 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -515,7 +515,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, "r"); + control_file = fopen(control, PG_BINARY_R); if (control_file == NULL) elog(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -561,7 +561,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, "w"); + control_file = fopen(control, PG_BINARY_W); if (control_file == NULL) elog(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); From ceaa2ce9cdf64b13f52df42f23a5640d339aeda5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 21 May 2018 19:06:12 +0300 Subject: [PATCH 28/66] PGPRO-533: Allow to show backup list in json format --- src/backup.c | 24 ++- src/catalog.c | 64 +++++- src/help.c | 2 + src/pg_probackup.c | 63 +++--- src/pg_probackup.h | 20 +- src/show.c | 494 +++++++++++++++++++++++++++++++++++++-------- src/util.c | 13 +- src/utils/pgut.h | 2 +- 8 files changed, 532 insertions(+), 150 deletions(-) diff --git a/src/backup.c b/src/backup.c index 57c4be51..6a2333bd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -633,7 +633,7 @@ do_backup_instance(void) * For backup from master wait for previous segment. * For backup from replica wait for current segment. */ - !from_replica, backup_files_list); + !current.from_replica, backup_files_list); } if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) @@ -815,11 +815,15 @@ do_backup(time_t start_time) pgut_atexit_push(backup_disconnect, NULL); current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); confirm_block_size("wal_block_size", XLOG_BLCKSZ); - from_replica = pg_is_in_recovery(); + current.from_replica = pg_is_in_recovery(); /* Confirm that this server version is supported */ check_server_version(); @@ -859,7 +863,7 @@ do_backup(time_t start_time) } } - if (from_replica) + if (current.from_replica) { /* Check master connection options */ if (master_host == NULL) @@ -956,7 +960,7 @@ check_server_version(void) "server version is %s, must be %s or higher", server_version_str, "9.5"); - if (from_replica && server_version < 90600) + if (current.from_replica && server_version < 90600) elog(ERROR, "server version is %s, must be %s or higher for backup from replica", server_version_str, "9.6"); @@ -1061,7 +1065,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; /* For replica we call pg_start_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (backup->from_replica) ? master_conn : backup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1106,7 +1110,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) } /* Wait for start_lsn to be replayed by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(backup->start_lsn, true); /* @@ -1554,8 +1558,6 @@ wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) { uint32 try_count = 0; - Assert(from_replica); - while (true) { PGresult *res; @@ -1650,7 +1652,7 @@ pg_stop_backup(pgBackup *backup) elog(FATAL, "backup is not in progress"); /* For replica we call pg_stop_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (current.from_replica) ? master_conn : backup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", @@ -1663,7 +1665,7 @@ pg_stop_backup(pgBackup *backup) const char *params[1]; char name[1024]; - if (!from_replica) + if (!current.from_replica) snprintf(name, lengthof(name), "pg_probackup, backup_id %s", base36enc(backup->start_time)); else @@ -1891,7 +1893,7 @@ pg_stop_backup(pgBackup *backup) stream_xlog_path[MAXPGPATH]; /* Wait for stop_lsn to be received by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. diff --git a/src/catalog.c b/src/catalog.c index f5884f01..6c9d36b5 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -385,10 +385,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "#Configuration\n"); fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); - fprintf(out, "stream = %s\n", backup->stream?"true":"false"); - fprintf(out, "compress-alg = %s\n", deparse_compress_alg(compress_alg)); - fprintf(out, "compress-level = %d\n", compress_level); - fprintf(out, "from-replica = %s\n", from_replica?"true":"false"); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); @@ -429,7 +430,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->data_bytes != BYTES_INVALID) fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); - if (backup->data_bytes != BYTES_INVALID) + if (backup->wal_bytes != BYTES_INVALID) fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); fprintf(out, "status = %s\n", status2str(backup->status)); @@ -475,10 +476,8 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; - char *compress_alg = NULL; char *server_version = NULL; - int *compress_level; - bool *from_replica; + char *compress_alg = NULL; pgut_option options[] = { @@ -500,8 +499,8 @@ readBackupControlFile(const char *path) {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, - {'u', 0, "compress-level", &compress_level, SOURCE_FILE_STRICT}, - {'b', 0, "from-replica", &from_replica, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {0} }; @@ -578,6 +577,9 @@ readBackupControlFile(const char *path) pfree(server_version); } + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + return backup; } @@ -626,6 +628,48 @@ deparse_backup_mode(BackupMode mode) return NULL; } +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + /* free pgBackup object */ void pgBackupFree(void *backup) diff --git a/src/help.c b/src/help.c index 2f84b225..a41e8167 100644 --- a/src/help.c +++ b/src/help.c @@ -128,6 +128,7 @@ help_pg_probackup(void) printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired]\n")); @@ -362,6 +363,7 @@ help_show(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5d464171..efb8e6a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -47,7 +47,6 @@ char *replication_slot = NULL; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -bool from_replica = false; bool is_remote_backup = false; /* Wait timeout for WAL segment archiving */ uint32 archive_timeout = 300; /* default is 300 seconds */ @@ -83,7 +82,7 @@ uint32 retention_window = 0; /* compression options */ CompressAlg compress_alg = NOT_DEFINED_COMPRESS; int compress_level = DEFAULT_COMPRESS_LEVEL; -bool compress_shortcut = false; +bool compress_shortcut = false; /* other options */ char *instance_name; @@ -94,6 +93,9 @@ static char *wal_file_path; static char *wal_file_name; static bool file_overwrite = false; +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + /* current settings */ pgBackup current; ProbackupSubcmd backup_subcmd; @@ -104,6 +106,7 @@ static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); static void compress_init(void); @@ -178,6 +181,8 @@ static pgut_option options[] = { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, { 0 } }; @@ -517,49 +522,31 @@ opt_log_level_file(pgut_option *opt, const char *arg) log_level_file = parse_log_level(arg); } -CompressAlg -parse_compress_alg(const char *arg) +static void +opt_show_format(pgut_option *opt, const char *arg) { + const char *v = arg; size_t len; /* Skip all spaces detected */ - while (isspace((unsigned char)*arg)) - arg++; - len = strlen(arg); + while (IsSpace(*v)) + v++; + len = strlen(v); - if (len == 0) - elog(ERROR, "compress algrorithm is empty"); - - if (pg_strncasecmp("zlib", arg, len) == 0) - return ZLIB_COMPRESS; - else if (pg_strncasecmp("pglz", arg, len) == 0) - return PGLZ_COMPRESS; - else if (pg_strncasecmp("none", arg, len) == 0) - return NONE_COMPRESS; - else - elog(ERROR, "invalid compress algorithm value \"%s\"", arg); - - return NOT_DEFINED_COMPRESS; -} - -const char* -deparse_compress_alg(int alg) -{ - switch (alg) + if (len > 0) { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - return "none"; - case ZLIB_COMPRESS: - return "zlib"; - case PGLZ_COMPRESS: - return "pglz"; + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } - - return NULL; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } -void +static void opt_compress_alg(pgut_option *opt, const char *arg) { compress_alg = parse_compress_alg(arg); @@ -568,8 +555,8 @@ opt_compress_alg(pgut_option *opt, const char *arg) /* * Initialize compress and sanity checks for compress. */ -static -void compress_init(void) +static void +compress_init(void) { /* Default algorithm is zlib */ if (compress_shortcut) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ce..0fefef1b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -149,6 +149,12 @@ typedef enum ProbackupSubcmd SHOW_CONFIG } ProbackupSubcmd; +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ @@ -214,6 +220,9 @@ typedef struct pgBackup /* Size of WAL files in archive needed to restore this backup */ int64 wal_bytes; + CompressAlg compress_alg; + int compress_level; + /* Fields needed for compatibility check */ uint32 block_size; uint32 wal_block_size; @@ -221,13 +230,14 @@ typedef struct pgBackup char server_version[100]; - bool stream; /* Was this backup taken in stream mode? + bool stream; /* Was this backup taken in stream mode? * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ time_t parent_backup; /* Identifier of the previous backup. * Which is basic backup for this * incremental backup. */ - char *primary_conninfo; /* Connection parameters of the backup - * in the format suitable for recovery.conf */ + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ } pgBackup; /* Recovery target for restore and validate subcommands */ @@ -310,7 +320,6 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; extern uint32 archive_timeout; -extern bool from_replica; extern bool is_remote_backup; extern const char *master_db; extern const char *master_host; @@ -348,6 +357,9 @@ extern const char* deparse_compress_alg(int alg); extern char *instance_name; extern uint64 system_identifier; +/* show options */ +extern ShowFormat show_format; + /* current settings */ extern pgBackup current; extern ProbackupSubcmd backup_subcmd; diff --git a/src/show.c b/src/show.c index b6eee867..b97d9427 100644 --- a/src/show.c +++ b/src/show.c @@ -3,28 +3,38 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" + #include #include #include #include +#include "pqexpbuffer.h" -static void show_backup_list(FILE *out, parray *backup_list); -static void show_backup_detail(FILE *out, pgBackup *backup); -static int do_show_instance(time_t requested_backup_id); + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static uint8 json_level = 0; int do_show(time_t requested_backup_id) { - - if (instance_name == NULL - && requested_backup_id != INVALID_BACKUP_ID) + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use --backup_id option"); if (instance_name == NULL) @@ -38,10 +48,12 @@ do_show(time_t requested_backup_id) join_path_components(path, backup_path, BACKUPS_DIR); dir = opendir(path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); - errno = 0; - while ((dent = readdir(dir))) + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) { char child[MAXPGPATH]; struct stat st; @@ -54,73 +66,47 @@ do_show(time_t requested_backup_id) join_path_components(child, path, dent->d_name); if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; instance_name = dent->d_name; sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - fprintf(stdout, "\nBACKUP INSTANCE '%s'\n", instance_name); - do_show_instance(0); + + show_instance(INVALID_BACKUP_ID, true); } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + return 0; } else - return do_show_instance(requested_backup_id); -} - -/* - * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information - * about all backups in the backup instance. - * If valid backup id is passed, show detailed meta information - * about specified backup. - */ -static int -do_show_instance(time_t requested_backup_id) -{ - if (requested_backup_id != INVALID_BACKUP_ID) - { - pgBackup *backup; - - backup = read_backup(requested_backup_id); - if (backup == NULL) - { - elog(INFO, "Requested backup \"%s\" is not found.", - /* We do not need free base36enc's result, we exit anyway */ - base36enc(requested_backup_id)); - /* This is not error */ - return 0; - } - - show_backup_detail(stdout, backup); - - /* cleanup */ - pgBackupFree(backup); - - } - else - { - parray *backup_list; - - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); - - show_backup_list(stdout, backup_list); - - /* cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } - - return 0; + return show_backup(requested_backup_id); } static void pretty_size(int64 size, char *buf, size_t len) { - int exp = 0; + int exp = 0; /* minus means the size is invalid */ if (size < 0) @@ -219,16 +205,113 @@ get_parent_tli(TimeLineID child_tli) return result; } +/* + * Initialize instance visualization. + */ static void -show_backup_list(FILE *out, parray *backup_list) +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + if (backup_list == NULL) + elog(ERROR, "Failed to get backup list."); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) { int i; + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + /* if you add new fields here, fix the header */ /* show header */ - fputs("============================================================================================================================================\n", out); - fputs(" Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); - fputs("============================================================================================================================================\n", out); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); for (i = 0; i < parray_num(backup_list); i++) { @@ -255,27 +338,270 @@ show_backup_list(FILE *out, parray *backup_list) /* Get parent timeline before printing */ parent_tli = get_parent_tli(backup->tli); - fprintf(out, " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", - instance_name, - (backup->server_version[0] ? backup->server_version : "----"), - base36enc(backup->start_time), - timestamp, - pgBackupGetBackupMode(backup), - backup->stream ? "STREAM": "ARCHIVE", - backup->tli, - parent_tli, - duration, - data_bytes_str, - (uint32) (backup->start_lsn >> 32), - (uint32) backup->start_lsn, - (uint32) (backup->stop_lsn >> 32), - (uint32) backup->stop_lsn, - status2str(backup->status)); + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +static void +json_add_indent(PQExpBuffer buf) +{ + uint8 i; + + if (json_level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < json_level; i++) + appendPQExpBufferStr(buf, " "); +} + +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +static void +json_add(PQExpBuffer buf, JsonToken type) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + json_level++; + break; + case JT_END_ARRAY: + json_level--; + if (json_level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf); + appendPQExpBufferChar(buf, '{'); + json_level++; + break; + case JT_END_OBJECT: + json_level--; + if (json_level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; } } static void -show_backup_detail(FILE *out, pgBackup *backup) +json_add_escaped(PQExpBuffer buf, const char *str) { - pgBackupWriteControl(out, backup); + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} + +static void +json_add_key(PQExpBuffer buf, const char *name, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +static void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + bool add_comma) +{ + json_add_key(buf, name, add_comma); + json_add_escaped(buf, value); +} + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT); + + json_add_value(buf, "instance-name", instance_name, false); + + json_add_key(buf, "backups", true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT); + + json_add_value(buf, "id", base36enc(backup->start_time), true); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), true); + + json_add_key(buf, "compress-level", true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", true); + + json_add_key(buf, "block-size", true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "server-version", backup->server_version, true); + + json_add_key(buf, "current-tli", true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, true); + + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, true); + + json_add_key(buf, "recovery-xid", true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, true); + + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + json_add_value(buf, "data-bytes", data_bytes_str, true); + + pretty_size(backup->wal_bytes, data_bytes_str, + lengthof(data_bytes_str)); + json_add_value(buf, "wal-bytes", data_bytes_str, true); + + if (backup->end_time != (time_t) 0) + { + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + json_add_value(buf, "time", duration, true); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, true); + + json_add_value(buf, "status", status2str(backup->status), true); + + json_add(buf, JT_END_OBJECT); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT); + + first_instance = false; } diff --git a/src/util.c b/src/util.c index f2c84f6e..7aeba211 100644 --- a/src/util.c +++ b/src/util.c @@ -176,8 +176,8 @@ uint32 get_data_checksum_version(bool safe) { ControlFileData ControlFile; - char *buffer; - size_t size; + char *buffer; + size_t size; /* First fetch file... */ buffer = slurpFile(pgdata, "global/pg_control", &size, safe); @@ -310,10 +310,19 @@ pgBackup_init(pgBackup *backup) backup->end_time = (time_t) 0; backup->recovery_xid = 0; backup->recovery_time = (time_t) 0; + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = NOT_DEFINED_COMPRESS; + backup->compress_level = 0; + backup->block_size = BLCKSZ; backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + backup->stream = false; + backup->from_replica = false; backup->parent_backup = 0; backup->primary_conninfo = NULL; backup->server_version[0] = '\0'; diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a9003f2f..803d2c57 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -59,7 +59,7 @@ typedef enum pgut_optsrc typedef struct pgut_option { char type; - char sname; /* short name */ + uint8 sname; /* short name */ const char *lname; /* long name */ void *var; /* pointer to variable */ pgut_optsrc allowed; /* allowed source */ From 6832cbbcb90c43a378e70771079dce2fad94b2d3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 22 May 2018 15:55:18 +0300 Subject: [PATCH 29/66] Replaced the backslash in the path to the backup --- src/restore.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/restore.c b/src/restore.c index d91678cb..71efaca5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -787,6 +787,14 @@ create_recovery_conf(time_t backup_id, fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); +#ifdef WIN32 + char *p1 = backup_path; + while (*p1){ + if ((*p1) == '\\') + *p1 = '/'; + p1++; + } +#endif fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s --wal-file-path %%p --wal-file-name %%f'\n", PROGRAM_NAME, backup_path, instance_name); fprintf(fp, "recovery_target_action = 'promote'\n"); From 6df8c2aaec213d0e149644df53606ab307cda336 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 31 May 2018 20:31:12 +0300 Subject: [PATCH 30/66] PGPRO-533: Replace instance-name by instance, fix typos --- src/show.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/show.c b/src/show.c index b97d9427..98dab419 100644 --- a/src/show.c +++ b/src/show.c @@ -494,7 +494,7 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); - json_add_value(buf, "instance-name", instance_name, false); + json_add_value(buf, "instance", instance_name, false); json_add_key(buf, "backups", true); @@ -517,13 +517,13 @@ show_instance_json(parray *backup_list) json_add(buf, JT_BEGIN_OBJECT); - json_add_value(buf, "id", base36enc(backup->start_time), true); + json_add_value(buf, "id", base36enc(backup->start_time), false); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", base36enc(backup->parent_backup), true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); From 733354c40f0e7540d781ea9240b728494d8b3119 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 1 Jun 2018 17:09:32 +0300 Subject: [PATCH 31/66] PGPRO-533: Reformat json output, add program-version to backup.control --- src/catalog.c | 10 ++++++++ src/pg_probackup.h | 1 + src/show.c | 58 ++++++++++++++++++++++++++++++++-------------- src/util.c | 1 + 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 6c9d36b5..eb5d3be7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -395,6 +395,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "block-size = %u\n", backup->block_size); fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); if (backup->server_version[0] != '\0') fprintf(out, "server-version = %s\n", backup->server_version); @@ -476,6 +477,7 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; + char *program_version = NULL; char *server_version = NULL; char *compress_alg = NULL; @@ -494,6 +496,7 @@ readBackupControlFile(const char *path) {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, @@ -570,6 +573,13 @@ readBackupControlFile(const char *path) free(parent_backup); } + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + if (server_version) { StrNCpy(backup->server_version, server_version, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0fefef1b..d3ce8241 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -228,6 +228,7 @@ typedef struct pgBackup uint32 wal_block_size; uint32 checksum_version; + char program_version[100]; char server_version[100]; bool stream; /* Was this backup taken in stream mode? diff --git a/src/show.c b/src/show.c index 98dab419..03ae6488 100644 --- a/src/show.c +++ b/src/show.c @@ -26,6 +26,21 @@ static int show_backup(time_t requested_backup_id); static void show_instance_plain(parray *backup_list, bool show_name); static void show_instance_json(parray *backup_list); +/* Json output functions */ + +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +static void json_add(PQExpBuffer buf, JsonToken type); +static void json_add_key(PQExpBuffer buf, const char *name, bool add_comma); +static void json_add_value(PQExpBuffer buf, const char *name, const char *value, + bool add_comma); + static PQExpBufferData show_buf; static bool first_instance = true; static uint8 json_level = 0; @@ -219,8 +234,9 @@ show_instance_start(void) first_instance = true; json_level = 0; - appendPQExpBufferChar(&show_buf, '['); - json_level++; + json_add(&show_buf, JT_BEGIN_OBJECT); + json_add_key(&show_buf, "instances", false); + json_add(&show_buf, JT_BEGIN_ARRAY); } /* @@ -230,7 +246,11 @@ static void show_instance_end(void) { if (show_format == SHOW_JSON) - appendPQExpBufferStr(&show_buf, "\n]\n"); + { + json_add(&show_buf, JT_END_ARRAY); + json_add(&show_buf, JT_END_OBJECT); + appendPQExpBufferChar(&show_buf, '\n'); + } fputs(show_buf.data, stdout); termPQExpBuffer(&show_buf); @@ -375,14 +395,6 @@ json_add_indent(PQExpBuffer buf) appendPQExpBufferStr(buf, " "); } -typedef enum -{ - JT_BEGIN_ARRAY, - JT_END_ARRAY, - JT_BEGIN_OBJECT, - JT_END_OBJECT -} JsonToken; - static void json_add(PQExpBuffer buf, JsonToken type) { @@ -493,10 +505,10 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, instance_name, false); - json_add_value(buf, "instance", instance_name, false); - - json_add_key(buf, "backups", true); + json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, "backups", false); /* * List backups. @@ -516,14 +528,19 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); json_add(buf, JT_BEGIN_OBJECT); + json_add_key(buf, base36enc(backup->start_time), false); - json_add_value(buf, "id", base36enc(backup->start_time), false); + /* Show backup attributes */ + json_add(buf, JT_BEGIN_OBJECT); if (backup->parent_backup != 0) + { json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), true); - - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + base36enc(backup->parent_backup), false); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + } + else + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); @@ -545,6 +562,7 @@ show_instance_json(parray *backup_list) json_add_key(buf, "checksum-version", true); appendPQExpBuffer(buf, "%u", backup->checksum_version); + json_add_value(buf, "program-version", backup->program_version, true); json_add_value(buf, "server-version", backup->server_version, true); json_add_key(buf, "current-tli", true); @@ -594,11 +612,15 @@ show_instance_json(parray *backup_list) json_add_value(buf, "status", status2str(backup->status), true); + json_add(buf, JT_END_OBJECT); + /* End of backup attributes */ + json_add(buf, JT_END_OBJECT); } /* End of backups */ json_add(buf, JT_END_ARRAY); + json_add(buf, JT_END_OBJECT); /* End of instance object */ json_add(buf, JT_END_OBJECT); diff --git a/src/util.c b/src/util.c index 7aeba211..a43239dc 100644 --- a/src/util.c +++ b/src/util.c @@ -325,5 +325,6 @@ pgBackup_init(pgBackup *backup) backup->from_replica = false; backup->parent_backup = 0; backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; } From 8aa559b17b24bc1d91f12df6df2362373fc372dd Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 1 Jun 2018 17:26:56 +0300 Subject: [PATCH 32/66] PGPRO-533: Reformat json output --- src/show.c | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/show.c b/src/show.c index 03ae6488..a39ba528 100644 --- a/src/show.c +++ b/src/show.c @@ -234,9 +234,8 @@ show_instance_start(void) first_instance = true; json_level = 0; - json_add(&show_buf, JT_BEGIN_OBJECT); - json_add_key(&show_buf, "instances", false); - json_add(&show_buf, JT_BEGIN_ARRAY); + appendPQExpBufferChar(&show_buf, '['); + json_level++; } /* @@ -246,11 +245,7 @@ static void show_instance_end(void) { if (show_format == SHOW_JSON) - { - json_add(&show_buf, JT_END_ARRAY); - json_add(&show_buf, JT_END_OBJECT); - appendPQExpBufferChar(&show_buf, '\n'); - } + appendPQExpBufferStr(&show_buf, "\n]\n"); fputs(show_buf.data, stdout); termPQExpBuffer(&show_buf); @@ -505,10 +500,9 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, instance_name, false); - json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, "backups", false); + json_add_value(buf, "instance", instance_name, false); + json_add_key(buf, "backups", true); /* * List backups. @@ -528,19 +522,14 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); json_add(buf, JT_BEGIN_OBJECT); - json_add_key(buf, base36enc(backup->start_time), false); - /* Show backup attributes */ - json_add(buf, JT_BEGIN_OBJECT); + json_add_value(buf, "id", base36enc(backup->start_time), false); if (backup->parent_backup != 0) - { json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), false); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); - } - else - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), false); + base36enc(backup->parent_backup), true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); @@ -612,15 +601,11 @@ show_instance_json(parray *backup_list) json_add_value(buf, "status", status2str(backup->status), true); - json_add(buf, JT_END_OBJECT); - /* End of backup attributes */ - json_add(buf, JT_END_OBJECT); } /* End of backups */ json_add(buf, JT_END_ARRAY); - json_add(buf, JT_END_OBJECT); /* End of instance object */ json_add(buf, JT_END_OBJECT); From 191d5e30e98d25a70f17ace622b4626f670e9b6d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Jun 2018 20:35:37 +0300 Subject: [PATCH 33/66] tests: json format for show command --- tests/archive.py | 14 +++- tests/backup_test.py | 38 +++++----- tests/delete_test.py | 30 ++++---- tests/delta.py | 4 +- tests/exclude.py | 18 +++-- tests/helpers/ptrack_helpers.py | 121 +++++++++++++++++++------------- tests/pgpro589.py | 2 +- tests/ptrack.py | 103 ++++++++++++++++++--------- tests/restore_test.py | 18 ++--- tests/retention_test.py | 12 ++-- tests/show_test.py | 29 ++++++++ tests/validate_test.py | 84 ++++++++++++---------- 12 files changed, 292 insertions(+), 181 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index c0408360..4e1f39d8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -427,7 +427,11 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): - """make node without archiving, take stream backup and turn it into replica, set replica with archiving, make archive backup from replica""" + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -468,7 +472,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, take FULL backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " @@ -502,7 +508,9 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, make PAGE backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " diff --git a/tests/backup_test.py b/tests/backup_test.py index 5af59684..1fa74643 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -29,15 +29,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - # full backup mode - # with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - # backup_log.write(self.backup_node(node, options=["--verbose"])) - backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "FULL") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied excluded = True @@ -61,29 +57,29 @@ class BackupTest(ProbackupTest, unittest.TestCase): # print self.show_pb(node) show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PAGE") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") # Check parent backup self.assertEqual( backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # ptrack backup mode self.backup_node(backup_dir, 'node', node, backup_type="ptrack") show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PTRACK") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") # Check parent backup self.assertEqual( page_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -106,7 +102,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") node.stop() # Clean after yourself @@ -162,7 +158,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") # Clean after yourself @@ -227,7 +223,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.assertEqual( self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR") + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") # Clean after yourself self.del_test_dir(module_name, fname) @@ -250,12 +246,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -282,11 +278,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -342,7 +338,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -415,7 +411,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/delete_test.py b/tests/delete_test.py index 039f3606..4afb15ae 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -44,13 +44,13 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node) show_backups = self.show_pb(backup_dir, 'node') - id_1 = show_backups[0]['ID'] - id_2 = show_backups[1]['ID'] - id_3 = show_backups[2]['ID'] + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] self.delete_pb(backup_dir, 'node', id_2) show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(show_backups[0]['ID'], id_1) - self.assertEqual(show_backups[1]['ID'], id_3) + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) # Clean after yourself self.del_test_dir(module_name, fname) @@ -82,15 +82,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -122,15 +122,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index 303c7c58..7cf21758 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1191,7 +1191,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -1264,7 +1264,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/exclude.py b/tests/exclude.py index 763060c7..4f9d73ab 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -34,25 +34,33 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): temp_schema_name = conn.execute("SELECT nspname FROM pg_namespace WHERE oid = pg_my_temp_schema()")[0][0] conn.commit() - temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace("pg_", "") + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") conn.commit() conn.execute("create index test_idx on test (generate_series)") conn.commit() - heap_path = conn.execute("select pg_relation_filepath('test')")[0][0] + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] conn.commit() - index_path = conn.execute("select pg_relation_filepath('test_idx')")[0][0] + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] conn.commit() heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] conn.commit() - toast_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] conn.commit() - toast_idx_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid) + "_index"))[0][0] + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] conn.commit() temp_table_filename = os.path.basename(heap_path) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a776599c..af7fe766 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -12,6 +12,7 @@ import select import psycopg2 from time import sleep import re +import json idx_ptrack = { 't_heap': { @@ -598,7 +599,7 @@ class ProbackupTest(object): def show_pb( self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False + options=[], as_text=False, as_json=True ): backup_list = [] @@ -613,63 +614,83 @@ class ProbackupTest(object): if backup_id: cmd_list += ["-i", backup_id] + if as_json: + cmd_list += ["--format=json"] + if as_text: # You should print it when calling as_text=true return self.run_pb(cmd_list + options) # get show result as list of lines - show_splitted = self.run_pb(cmd_list + options).splitlines() - if instance is not None and backup_id is None: - # cut header(ID, Mode, etc) from show as single string - header = show_splitted[1:2][0] - # cut backup records from show as single list - # with string for every backup record - body = show_splitted[3:] - # inverse list so oldest record come first - body = body[::-1] - # split string in list with string for every header element - header_split = re.split(" +", header) - # Remove empty items - for i in header_split: - if i == '': - header_split.remove(i) + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: continue - header_split = [ - header_element.rstrip() for header_element in header_split - ] - for backup_record in body: - backup_record = backup_record.rstrip() - # split list with str for every backup record element - backup_record_split = re.split(" +", backup_record) - # Remove empty items - for i in backup_record_split: - if i == '': - backup_record_split.remove(i) - if len(header_split) != len(backup_record_split): - print(warning.format( - header=header, body=body, - header_split=header_split, - body_split=backup_record_split) - ) - exit(1) - new_dict = dict(zip(header_split, backup_record_split)) - backup_list.append(new_dict) + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) return backup_list else: - # cut out empty lines and lines started with # - # and other garbage then reconstruct it as dictionary - # print show_splitted - sanitized_show = [item for item in show_splitted if item] - sanitized_show = [ - item for item in sanitized_show if not item.startswith('#') - ] - # print sanitized_show - for line in sanitized_show: - name, var = line.partition(" = ")[::2] - var = var.strip('"') - var = var.strip("'") - specific_record[name.strip()] = var - return specific_record + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record def validate_pb( self, backup_dir, instance=None, diff --git a/tests/pgpro589.py b/tests/pgpro589.py index a67f3dd4..bd40f16d 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -63,7 +63,7 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] self.assertEqual( 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') diff --git a/tests/ptrack.py b/tests/ptrack.py index 9c21ff55..4823acef 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -268,7 +268,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -430,7 +431,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -503,7 +505,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() @@ -584,8 +587,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd) ) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) @@ -604,12 +607,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -688,9 +692,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd) ) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -711,12 +717,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -811,7 +818,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_pgpro417(self): - """Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail""" + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -880,7 +890,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_full_pgpro417(self): - """Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail""" + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -954,7 +967,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_create_db(self): - """Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense""" + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1017,7 +1033,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1046,7 +1063,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1151,7 +1169,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET RESTORED PGDATA AND COMPARE if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1159,8 +1178,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) node_restored.start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node_restored.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1229,7 +1248,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1240,7 +1260,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_drop_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1321,7 +1344,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1379,15 +1405,16 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while restored_node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: time.sleep(1) # COMPARE LOGICAL CONTENT @@ -1416,14 +1443,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( + while restored_node.psql( "postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) @@ -1437,7 +1465,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_multiple_segments(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1446,9 +1477,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '128MB', - 'maintenance_work_mem': '1GB', 'autovacuum': 'off', - 'full_page_writes': 'off'} + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } ) self.init_pb(backup_dir) @@ -1514,14 +1546,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM NODE_RESTORED if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) restored_node.start() - while restored_node.safe_psql( + while restored_node.psql( "postgres", "select pg_is_in_recovery()") == 't\n': time.sleep(1) diff --git a/tests/restore_test.py b/tests/restore_test.py index 4567f37b..fce96911 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -718,7 +718,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # 1 - Try to restore to existing directory node.stop() @@ -785,8 +785,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="page") show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['Status'], "OK") - self.assertEqual(show_pb[2]['Status'], "OK") + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") node.stop() node.cleanup() @@ -829,7 +829,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Full backup self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -845,8 +845,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # First page backup self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: @@ -862,8 +863,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Second page backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") node.stop() node.cleanup() diff --git a/tests/retention_test.py b/tests/retention_test.py index 2d4cac37..652f7c39 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -14,7 +14,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -24,7 +25,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") # Make backups to be purged @@ -57,7 +60,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - #wal_name_b = wal_name.encode('ascii') + # wal_name_b = wal_name.encode('ascii') self.assertEqual(wal_name[8:] > min_wal[8:], True) self.assertEqual(wal_name[8:] > max_wal[8:], True) @@ -68,7 +71,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/show_test.py b/tests/show_test.py index 74bd0341..29d0bdb3 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -36,6 +36,35 @@ class OptionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" diff --git a/tests/validate_test.py b/tests/validate_test.py index 06ea1ea3..afb2305f 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -908,7 +908,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) target_xid = None with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() target_xid = res[0][0] @@ -1041,7 +1042,10 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): - """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), @@ -1083,7 +1087,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): else: walfile = node.safe_psql( 'postgres', - 'select pg_walfile_name(pg_current_wal_location())').rstrip() + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -1134,12 +1138,12 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup STATUS should be "OK"') # Clean after yourself @@ -1208,7 +1212,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Clean after yourself @@ -1405,7 +1409,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: pass self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') self.set_archiving(backup_dir, 'node', node) node.reload() self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1440,14 +1444,19 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') os.rename(file_new, file) try: @@ -1459,14 +1468,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') # Clean after yourself self.del_test_dir(module_name, fname) @@ -1537,13 +1547,13 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') os.rename(file_new, file) file = os.path.join( @@ -1562,13 +1572,13 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') # Clean after yourself self.del_test_dir(module_name, fname) From c71151d3df99b5b656190da7225e48ab82c01d7c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Jun 2018 14:07:10 +0300 Subject: [PATCH 34/66] PGPRO-533: Add json format for show-config --- Makefile | 3 +- src/configure.c | 162 ++++++++++++++++++++++++++++++++- src/show.c | 232 ++++++++++++----------------------------------- src/utils/json.c | 134 +++++++++++++++++++++++++++ src/utils/json.h | 33 +++++++ 5 files changed, 388 insertions(+), 176 deletions(-) create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h diff --git a/Makefile b/Makefile index 0f4fa672..9880e5db 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/pg_probackup.o src/restore.o src/show.o src/status.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ - src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h diff --git a/src/configure.c b/src/configure.c index 55d2bbc5..0410dc74 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,19 +2,33 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" +#include "pqexpbuffer.h" + +#include "utils/json.h" + + static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + static pgBackupConfig *cur_config = NULL; +static PQExpBufferData show_buf; +static int32 json_level = 0; + /* Set configure options */ int do_configure(bool show_only) @@ -68,7 +82,7 @@ do_configure(bool show_only) config->compress_level = compress_level; if (show_only) - writeBackupCatalogConfig(stderr, config); + show_configure(config); else writeBackupCatalogConfigFile(config); @@ -251,7 +265,6 @@ readBackupCatalogConfigFile(void) pgut_readopt(path, options, ERROR); return config; - } static void @@ -271,3 +284,146 @@ opt_compress_alg(pgut_option *opt, const char *arg) { cur_config->compress_alg = parse_compress_alg(arg); } + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + if (config->replica_timeout != INT_MIN) + { + json_add_key(buf, "replica-timeout", json_level, true); + appendPQExpBuffer(buf, "%d", config->replica_timeout); + } + + /* Logging parameters */ + if (config->log_level_console != INT_MIN) + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + if (config->log_level_file != INT_MIN) + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + if (config->log_filename) + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + if (config->log_directory) + json_add_value(buf, "log-directory", config->log_directory, json_level, + true); + + if (config->log_rotation_size) + { + json_add_key(buf, "log-rotation-size", json_level, true); + appendPQExpBuffer(buf, "%d", config->log_rotation_size); + } + if (config->log_rotation_age) + { + json_add_key(buf, "log-rotation-age", json_level, true); + appendPQExpBuffer(buf, "%d", config->log_rotation_age); + } + + /* Retention parameters */ + if (config->retention_redundancy) + { + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + } + if (config->retention_window) + { + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + } + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/show.c b/src/show.c index a39ba528..01a558e0 100644 --- a/src/show.c +++ b/src/show.c @@ -17,6 +17,8 @@ #include "pqexpbuffer.h" +#include "utils/json.h" + static void show_instance_start(void); static void show_instance_end(void); @@ -26,24 +28,9 @@ static int show_backup(time_t requested_backup_id); static void show_instance_plain(parray *backup_list, bool show_name); static void show_instance_json(parray *backup_list); -/* Json output functions */ - -typedef enum -{ - JT_BEGIN_ARRAY, - JT_END_ARRAY, - JT_BEGIN_OBJECT, - JT_END_OBJECT -} JsonToken; - -static void json_add(PQExpBuffer buf, JsonToken type); -static void json_add_key(PQExpBuffer buf, const char *name, bool add_comma); -static void json_add_value(PQExpBuffer buf, const char *name, const char *value, - bool add_comma); - static PQExpBufferData show_buf; static bool first_instance = true; -static uint8 json_level = 0; +static int32 json_level = 0; int do_show(time_t requested_backup_id) @@ -377,115 +364,6 @@ show_instance_plain(parray *backup_list, bool show_name) * Json output. */ -static void -json_add_indent(PQExpBuffer buf) -{ - uint8 i; - - if (json_level == 0) - return; - - appendPQExpBufferChar(buf, '\n'); - for (i = 0; i < json_level; i++) - appendPQExpBufferStr(buf, " "); -} - -static void -json_add(PQExpBuffer buf, JsonToken type) -{ - switch (type) - { - case JT_BEGIN_ARRAY: - appendPQExpBufferChar(buf, '['); - json_level++; - break; - case JT_END_ARRAY: - json_level--; - if (json_level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf); - appendPQExpBufferChar(buf, ']'); - break; - case JT_BEGIN_OBJECT: - json_add_indent(buf); - appendPQExpBufferChar(buf, '{'); - json_level++; - break; - case JT_END_OBJECT: - json_level--; - if (json_level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf); - appendPQExpBufferChar(buf, '}'); - break; - default: - break; - } -} - -static void -json_add_escaped(PQExpBuffer buf, const char *str) -{ - const char *p; - - appendPQExpBufferChar(buf, '"'); - for (p = str; *p; p++) - { - switch (*p) - { - case '\b': - appendPQExpBufferStr(buf, "\\b"); - break; - case '\f': - appendPQExpBufferStr(buf, "\\f"); - break; - case '\n': - appendPQExpBufferStr(buf, "\\n"); - break; - case '\r': - appendPQExpBufferStr(buf, "\\r"); - break; - case '\t': - appendPQExpBufferStr(buf, "\\t"); - break; - case '"': - appendPQExpBufferStr(buf, "\\\""); - break; - case '\\': - appendPQExpBufferStr(buf, "\\\\"); - break; - default: - if ((unsigned char) *p < ' ') - appendPQExpBuffer(buf, "\\u%04x", (int) *p); - else - appendPQExpBufferChar(buf, *p); - break; - } - } - appendPQExpBufferChar(buf, '"'); -} - -static void -json_add_key(PQExpBuffer buf, const char *name, bool add_comma) -{ - if (add_comma) - appendPQExpBufferChar(buf, ','); - json_add_indent(buf); - - json_add_escaped(buf, name); - appendPQExpBufferStr(buf, ": "); -} - -static void -json_add_value(PQExpBuffer buf, const char *name, const char *value, - bool add_comma) -{ - json_add_key(buf, name, add_comma); - json_add_escaped(buf, value); -} - /* * Show instance backups in json format. */ @@ -499,116 +377,126 @@ show_instance_json(parray *backup_list) appendPQExpBufferChar(buf, ','); /* Begin of instance object */ - json_add(buf, JT_BEGIN_OBJECT); + json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "instance", instance_name, false); - json_add_key(buf, "backups", true); + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); /* * List backups. */ - json_add(buf, JT_BEGIN_ARRAY); + json_add(buf, JT_BEGIN_ARRAY, &json_level); for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = parray_get(backup_list, i); TimeLineID parent_tli; char timestamp[100] = "----"; - char duration[20] = "----"; - char data_bytes_str[10] = "----"; char lsn[20]; if (i != 0) appendPQExpBufferChar(buf, ','); - json_add(buf, JT_BEGIN_OBJECT); + json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "id", base36enc(backup->start_time), false); + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), true); + base36enc(backup->parent_backup), json_level, true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), true); + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); - json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", true); + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); json_add_value(buf, "compress-alg", - deparse_compress_alg(backup->compress_alg), true); + deparse_compress_alg(backup->compress_alg), json_level, + true); - json_add_key(buf, "compress-level", true); + json_add_key(buf, "compress-level", json_level, true); appendPQExpBuffer(buf, "%d", backup->compress_level); json_add_value(buf, "from-replica", - backup->from_replica ? "true" : "false", true); + backup->from_replica ? "true" : "false", json_level, + true); - json_add_key(buf, "block-size", true); + json_add_key(buf, "block-size", json_level, true); appendPQExpBuffer(buf, "%u", backup->block_size); - json_add_key(buf, "xlog-block-size", true); + json_add_key(buf, "xlog-block-size", json_level, true); appendPQExpBuffer(buf, "%u", backup->wal_block_size); - json_add_key(buf, "checksum-version", true); + json_add_key(buf, "checksum-version", json_level, true); appendPQExpBuffer(buf, "%u", backup->checksum_version); - json_add_value(buf, "program-version", backup->program_version, true); - json_add_value(buf, "server-version", backup->server_version, true); + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); - json_add_key(buf, "current-tli", true); + json_add_key(buf, "current-tli", json_level, true); appendPQExpBuffer(buf, "%d", backup->tli); - json_add_key(buf, "parent-tli", true); + json_add_key(buf, "parent-tli", json_level, true); parent_tli = get_parent_tli(backup->tli); appendPQExpBuffer(buf, "%u", parent_tli); snprintf(lsn, lengthof(lsn), "%X/%X", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); - json_add_value(buf, "start-lsn", lsn, true); + json_add_value(buf, "start-lsn", lsn, json_level, true); snprintf(lsn, lengthof(lsn), "%X/%X", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - json_add_value(buf, "stop-lsn", lsn, true); + json_add_value(buf, "stop-lsn", lsn, json_level, true); time2iso(timestamp, lengthof(timestamp), backup->start_time); - json_add_value(buf, "start-time", timestamp, true); + json_add_value(buf, "start-time", timestamp, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->end_time); - json_add_value(buf, "end-time", timestamp, true); + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } - json_add_key(buf, "recovery-xid", true); + json_add_key(buf, "recovery-xid", json_level, true); appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - json_add_value(buf, "recovery-time", timestamp, true); - - pretty_size(backup->data_bytes, data_bytes_str, - lengthof(data_bytes_str)); - json_add_value(buf, "data-bytes", data_bytes_str, true); - - pretty_size(backup->wal_bytes, data_bytes_str, - lengthof(data_bytes_str)); - json_add_value(buf, "wal-bytes", data_bytes_str, true); - - if (backup->end_time != (time_t) 0) + if (backup->recovery_time > 0) { - snprintf(duration, lengthof(duration), "%.*lfs", 0, - difftime(backup->end_time, backup->start_time)); - json_add_value(buf, "time", duration, true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); } if (backup->primary_conninfo) - json_add_value(buf, "primary_conninfo", backup->primary_conninfo, true); + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); - json_add_value(buf, "status", status2str(backup->status), true); + json_add_value(buf, "status", status2str(backup->status), json_level, + true); - json_add(buf, JT_END_OBJECT); + json_add(buf, JT_END_OBJECT, &json_level); } /* End of backups */ - json_add(buf, JT_END_ARRAY); + json_add(buf, JT_END_ARRAY, &json_level); /* End of instance object */ - json_add(buf, JT_END_OBJECT); + json_add(buf, JT_END_OBJECT, &json_level); first_instance = false; } diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 00000000..3afbe9e7 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 00000000..cf5a7064 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ From efc0c8cda31dc58c43379b018136e0c0c7b458ae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 Jun 2018 12:22:45 +0300 Subject: [PATCH 35/66] bugfix: WAL delivery timeout could cause PostgreSQL instance to be left in in_backup state --- src/backup.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index a0b02fcf..f0f7ac9c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1076,6 +1076,12 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) 2, params); + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ @@ -1108,12 +1114,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* Wait for start_lsn to be replayed by replica */ if (from_replica) wait_replica_wal_lsn(backup->start_lsn, true); - - /* - * Set flag that pg_start_backup() was called. If an error will happen it - * is necessary to call pg_stop_backup() in backup_cleanup(). - */ - backup_in_progress = true; } /* From 9cd12fbe055b0bf1e494357713b88d2c1c622680 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 7 Jun 2018 14:54:25 +0300 Subject: [PATCH 36/66] PGPRO-533: Update hellp command --- src/configure.c | 4 ++++ src/help.c | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/configure.c b/src/configure.c index 0410dc74..dc80981a 100644 --- a/src/configure.c +++ b/src/configure.c @@ -29,6 +29,10 @@ static pgBackupConfig *cur_config = NULL; static PQExpBufferData show_buf; static int32 json_level = 0; +/* + * All this code needs refactoring. + */ + /* Set configure options */ int do_configure(bool show_only) diff --git a/src/help.c b/src/help.c index a41e8167..28500e75 100644 --- a/src/help.c +++ b/src/help.c @@ -89,6 +89,7 @@ help_pg_probackup(void) printf(_(" [--replica-timeout=timeout]\n")); printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); @@ -358,7 +359,8 @@ static void help_show(void) { printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n\n")); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); @@ -474,10 +476,12 @@ help_set_config(void) static void help_show_config(void) { - printf(_("%s show-config -B backup-dir --instance=instance_name\n\n"), PROGRAM_NAME); + printf(_("%s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void From e36daf5d1d8d42cd5413715e60b67328e701177b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 7 Jun 2018 19:13:11 +0300 Subject: [PATCH 37/66] Merge master into pg_probackup_windows --- Makefile | 3 +- doit.cmd | 1 + doit96.cmd | 1 + msvs/pg_probackup.sln | 28 ++++ msvs/template.pg_probackup.vcxproj | 209 +++++++++++++++++++++++ msvs/template.pg_probackup96.vcxproj | 207 +++++++++++++++++++++++ msvs/template.pg_probackup_2.vcxproj | 203 ++++++++++++++++++++++ src/backup.c | 73 ++++---- src/catalog.c | 1 - src/configure.c | 3 +- src/data.c | 56 +++---- src/dir.c | 18 +- src/pg_probackup.c | 15 +- src/pg_probackup.h | 27 ++- src/restore.c | 72 ++++---- src/status.c | 2 +- src/utils/logger.c | 22 ++- src/utils/pgut.c | 12 +- src/utils/pgut.h | 8 - src/utils/thread.c | 80 +++++++++ src/utils/thread.h | 32 ++++ src/validate.c | 44 +++-- win32build.pl | 240 +++++++++++++++++++++++++++ win32build96.pl | 240 +++++++++++++++++++++++++++ win32build_2.pl | 219 ++++++++++++++++++++++++ 25 files changed, 1664 insertions(+), 152 deletions(-) create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/Makefile b/Makefile index 9880e5db..eb3e3ee2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ - src/utils/json.o + src/utils/json.o src/utils/thread.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h @@ -63,6 +63,7 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..cc29bc88 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..044a3e73 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,209 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..c095d7cc --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,207 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..2fc101a4 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 5b4063fa..f24a0b0f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -18,16 +18,16 @@ #include #include #include -#include -#include "libpq/pqsignal.h" -#include "storage/bufpage.h" #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" #include "datapagemap.h" -#include "receivelog.h" -#include "streamutil.h" +#include "libpq/pqsignal.h" #include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; @@ -89,8 +89,8 @@ static bool pg_stop_backup_is_sent = false; static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -static void backup_files(void *arg); -static void remote_backup_files(void *arg); +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); static void do_backup_instance(void); @@ -253,7 +253,11 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) else if (copybuf[156] == '2') { /* Symlink */ +#ifndef WIN32 pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif } else elog(ERROR, "Unrecognized link indicator \"%c\"\n", @@ -289,7 +293,7 @@ remote_copy_file(PGconn *conn, pgFile* file) DATABASE_DIR); join_path_components(to_path, database_path, file->path); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -363,7 +367,7 @@ remote_copy_file(PGconn *conn, pgFile* file) * Take a remote backup of the PGDATA at a file level. * Copy all directories and files listed in backup_files_list. */ -static void +static void * remote_backup_files(void *arg) { int i; @@ -385,7 +389,7 @@ remote_backup_files(void *arg) if (S_ISDIR(file->mode)) continue; - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); @@ -441,6 +445,8 @@ remote_backup_files(void *arg) /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* @@ -456,8 +462,9 @@ do_backup_instance(void) char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - pthread_t backup_threads[num_threads]; - backup_files_args *backup_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *backup_threads; + backup_files_args *backup_threads_args; bool backup_isok = true; pgBackup *prev_backup = NULL; @@ -681,16 +688,19 @@ do_backup_instance(void) } /* setup threads */ - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); /* init thread args with own file lists */ + backup_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + backup_threads_args = (backup_files_args *) palloc(sizeof(backup_files_args)*num_threads); + for (i = 0; i < num_threads; i++) { - backup_files_args *arg = pg_malloc(sizeof(backup_files_args)); + backup_files_args *arg = &(backup_threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; @@ -701,33 +711,27 @@ do_backup_instance(void) arg->thread_cancel_conn = NULL; /* By default there are some error */ arg->ret = 1; - backup_threads_args[i] = arg; } /* Run threads */ elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { + backup_files_args *arg = &(backup_threads_args[i]); elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) backup_files, - backup_threads_args[i]); + pthread_create(&backup_threads[i], NULL, backup_files, arg); else - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) remote_backup_files, - backup_threads_args[i]); + pthread_create(&backup_threads[i], NULL, remote_backup_files, arg); } - + /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); - if (backup_threads_args[i]->ret == 1) + if (backup_threads_args[i].ret == 1) backup_isok = false; - - pg_free(backup_threads_args[i]); } if (backup_isok) elog(LOG, "Data files are transfered"); @@ -1017,7 +1021,7 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - + if (system_id_conn != system_identifier) elog(ERROR, "Backup data directory was initialized for system id %ld, but connected instance system id is %ld", system_identifier, system_id_conn); @@ -1040,14 +1044,15 @@ confirm_block_size(const char *name, int blcksz) res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); - + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - PQclear(res); - if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); + + PQclear(res);//bad pointer to endp + } /* @@ -1802,7 +1807,7 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, "w"); + fp = fopen(backup_label, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1850,7 +1855,7 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, "w"); + fp = fopen(tablespace_map, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); @@ -2006,7 +2011,7 @@ backup_disconnect(bool fatal, void *userdata) * In incremental backup mode, copy only files or datafiles' pages changed after * previous backup. */ -static void +static void * backup_files(void *arg) { int i; @@ -2021,7 +2026,7 @@ backup_files(void *arg) pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ @@ -2126,6 +2131,8 @@ backup_files(void *arg) /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* diff --git a/src/catalog.c b/src/catalog.c index eb5d3be7..f19628c3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/src/configure.c b/src/configure.c index dc80981a..2fc27892 100644 --- a/src/configure.c +++ b/src/configure.c @@ -132,7 +132,8 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); + //fprintf(out, "system-identifier = %li\n", config->system_identifier); + fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/data.c b/src/data.c index a1bdb91a..be07ce64 100644 --- a/src/data.c +++ b/src/data.c @@ -446,7 +446,7 @@ backup_data_file(backup_files_args* arguments, INIT_CRC32C(file->crc); /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(file->crc); @@ -480,7 +480,7 @@ backup_data_file(backup_files_args* arguments, /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -569,18 +569,18 @@ restore_data_file(const char *from_root, pgFile *file, pgBackup *backup) { - char to_path[MAXPGPATH]; - FILE *in = NULL; - FILE *out = NULL; - BackupPageHeader header; - BlockNumber blknum; - size_t file_size; + char to_path[MAXPGPATH]; + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum; + size_t file_size; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) { /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { elog(ERROR, "cannot open backup file \"%s\": %s", file->path, @@ -594,9 +594,9 @@ restore_data_file(const char *from_root, * re-open it with "w" to create an empty file. */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "r+"); + out = fopen(to_path, PG_BINARY_R "+"); if (out == NULL && errno == ENOENT) - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -690,13 +690,13 @@ restore_data_file(const char *from_root, } } - /* - * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do - * But during DELTA backup we read every file in PGDATA and thus DELTA backup - * knows exact size of every file at the time of backup. - * So when restoring file from DELTA backup we, knowning it`s size at - * a time of a backup, can truncate file to this size. - */ + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA) { @@ -759,7 +759,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); @@ -775,7 +775,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -918,7 +918,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out; + FILE *out=NULL; char buf[XLOG_BLCKSZ]; const char *to_path_p = to_path; char to_path_temp[MAXPGPATH]; @@ -930,7 +930,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -946,7 +946,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = gzopen(to_path_temp, "wb"); + gz_out = gzopen(to_path_temp, PG_BINARY_W); if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", compress_level, to_path_temp, get_gz_error(gz_out, errno)); @@ -961,7 +961,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1083,7 +1083,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) { #ifdef HAVE_LIBZ @@ -1092,7 +1092,7 @@ get_wal_file(const char *from_path, const char *to_path) * extension. */ snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - gz_in = gzopen(gz_from_path, "rb"); + gz_in = gzopen(gz_from_path, PG_BINARY_R); if (gz_in == NULL) { if (errno == ENOENT) @@ -1120,7 +1120,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1254,7 +1254,7 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); diff --git a/src/dir.c b/src/dir.c index 8df3da3f..20725f88 100644 --- a/src/dir.c +++ b/src/dir.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include #include #include @@ -102,10 +101,12 @@ int dir_create_dir(const char *dir, mode_t mode) { char copy[MAXPGPATH]; - char parent[MAXPGPATH]; + char *parent; strncpy(copy, dir, MAXPGPATH); - strncpy(parent, dirname(copy), MAXPGPATH); + + parent = pstrdup(dir); + get_parent_directory(parent); /* Create parent first */ if (access(parent, F_OK) == -1) @@ -119,6 +120,7 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } + pfree(parent); return 0; } @@ -153,6 +155,8 @@ pgFileInit(const char *path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); + file->name = 0; + file->size = 0; file->mode = 0; file->read_size = 0; @@ -232,7 +236,7 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, "r"); + fp = fopen(file->path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", file->path, strerror(errno)); @@ -350,7 +354,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, "r"); + black_list_file = fopen(path, PG_BINARY_R); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); @@ -827,7 +831,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); +#ifndef WIN32 if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != -1) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index efb8e6a3..c66179e1 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -10,6 +10,7 @@ #include "pg_probackup.h" #include "streamutil.h" +#include "utils/thread.h" #include #include @@ -207,11 +208,7 @@ main(int argc, char *argv[]) /* * Save main thread's tid. It is used call exit() in case of errors. */ -#ifdef WIN32 - main_tid = GetCurrentThreadId(); -#else main_tid = pthread_self(); -#endif /* Parse subcommands and non-subcommand options */ if (argc > 1) @@ -235,7 +232,7 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "show") == 0) backup_subcmd = SHOW; else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; + backup_subcmd = DELETE_SUBCMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG; else if (strcmp(argv[1], "show-config") == 0) @@ -281,7 +278,7 @@ main(int argc, char *argv[]) if (backup_subcmd == BACKUP || backup_subcmd == RESTORE || backup_subcmd == VALIDATE || - backup_subcmd == DELETE) + backup_subcmd == DELETE_SUBCMD) { int i, len = 0, @@ -325,6 +322,7 @@ main(int argc, char *argv[]) if (backup_path == NULL) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } + canonicalize_path(backup_path); /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) @@ -402,7 +400,7 @@ main(int argc, char *argv[]) { if (backup_subcmd != RESTORE && backup_subcmd != VALIDATE - && backup_subcmd != DELETE + && backup_subcmd != DELETE_SUBCMD && backup_subcmd != SHOW) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -464,6 +462,7 @@ main(int argc, char *argv[]) start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, @@ -484,7 +483,7 @@ main(int argc, char *argv[]) false); case SHOW: return do_show(current.backup_id); - case DELETE: + case DELETE_SUBCMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d3ce8241..8face84d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -15,10 +15,6 @@ #include #include -#ifndef WIN32 -#include -#endif - #include "access/timeline.h" #include "access/xlogdefs.h" #include "access/xlog_internal.h" @@ -28,6 +24,13 @@ #include "storage/checksum.h" #include "utils/pg_crc.h" #include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif #include "utils/parray.h" #include "utils/pgut.h" @@ -102,7 +105,7 @@ typedef struct pgFile bool is_database; bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile uint32 lock; /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; @@ -144,7 +147,7 @@ typedef enum ProbackupSubcmd RESTORE, VALIDATE, SHOW, - DELETE, + DELETE_SUBCMD, SET_CONFIG, SHOW_CONFIG } ProbackupSubcmd; @@ -531,4 +534,16 @@ extern void pgBackup_init(pgBackup *backup); /* in status.c */ extern bool is_pg_running(void); +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + #endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c index 69bac841..0e42c4c4 100644 --- a/src/restore.c +++ b/src/restore.c @@ -14,14 +14,14 @@ #include #include #include -#include #include "catalog/pg_control.h" +#include "utils/thread.h" typedef struct { - parray *files; - pgBackup *backup; + parray *files; + pgBackup *backup; /* * Return value from the thread. @@ -65,7 +65,7 @@ static void check_tablespace_mapping(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); -static void restore_files(void *arg); +static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); static const char *get_tablespace_mapping(const char *dir); static void set_tablespace_created(const char *link, const char *dir); @@ -80,13 +80,11 @@ static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int -do_restore_or_validate(time_t target_backup_id, - pgRecoveryTarget *rt, - bool is_restore) +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) { int i; parray *backups; - parray *timelines; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; @@ -169,6 +167,8 @@ do_restore_or_validate(time_t target_backup_id, if (rt->recovery_target_tli) { + parray *timelines; + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); /* Read timeline history files from archives */ timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); @@ -362,8 +362,9 @@ restore_backup(pgBackup *backup) char list_path[MAXPGPATH]; parray *files; int i; - pthread_t restore_threads[num_threads]; - restore_files_args *restore_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *restore_threads; + restore_files_args *restore_threads_args; bool restore_isok = true; if (backup->status != BACKUP_STATUS_OK) @@ -397,17 +398,21 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); files = dir_read_file_list(database_path, list_path); + restore_threads = (pthread_t *) palloc(sizeof(pthread_t)*num_threads); + restore_threads_args = (restore_files_args *) palloc(sizeof(restore_files_args)*num_threads); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = pg_malloc(sizeof(restore_files_args)); + restore_files_args *arg = &(restore_threads_args[i]); + arg->files = files; arg->backup = backup; /* By default there are some error */ @@ -415,23 +420,22 @@ restore_backup(pgBackup *backup) elog(LOG, "Start thread for num:%li", parray_num(files)); - restore_threads_args[i] = arg; - pthread_create(&restore_threads[i], NULL, - (void *(*)(void *)) restore_files, arg); + pthread_create(&restore_threads[i], NULL, restore_files, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { pthread_join(restore_threads[i], NULL); - if (restore_threads_args[i]->ret == 1) + if (restore_threads_args[i].ret == 1) restore_isok = false; - - pg_free(restore_threads_args[i]); } if (!restore_isok) elog(ERROR, "Data files restoring failed"); + pfree(restore_threads); + pfree(restore_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); @@ -452,7 +456,7 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - int i; + int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ @@ -702,7 +706,7 @@ check_tablespace_mapping(pgBackup *backup) /* * Restore files into $PGDATA. */ -static void +static void * restore_files(void *arg) { int i; @@ -714,7 +718,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, @@ -779,6 +783,8 @@ restore_files(void *arg) /* Data files restoring is successful */ arguments->ret = 0; + + return NULL; } /* Create recovery.conf with given recovery target parameters */ @@ -787,9 +793,9 @@ create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup) { - char path[MAXPGPATH]; - FILE *fp; - bool need_restore_conf = false; + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; if (!backup->stream || (rt->time_specified || rt->xid_specified)) @@ -959,7 +965,8 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - entry->end = (uint32) (-1UL << 32) | -1UL; + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; parray_insert(result, 0, entry); return result; @@ -980,10 +987,13 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) bool satisfy_timeline(const parray *timelines, const pgBackup *backup) { - int i; + int i; + for (i = 0; i < parray_num(timelines); i++) { - TimeLineHistoryEntry *timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && backup->stop_lsn < timeline->end) return true; @@ -1003,14 +1013,14 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_name, const char *target_action) { - time_t dummy_time; - TransactionId dummy_xid; - bool dummy_bool; + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; /* * count the number of the mutually exclusive options which may specify * recovery target. If final value > 1, throw an error. */ - int recovery_target_specified = 0; + int recovery_target_specified = 0; pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); /* fill all options with default values */ diff --git a/src/status.c b/src/status.c index 1c7c6038..155a07f4 100644 --- a/src/status.c +++ b/src/status.c @@ -38,7 +38,7 @@ get_pgpid(void) snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); - pidf = fopen(pid_file, "r"); + pidf = fopen(pid_file, PG_BINARY_R); if (pidf == NULL) { /* No pid file, not an error on startup */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 0a4f835b..8fd78739 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -16,6 +15,7 @@ #include "logger.h" #include "pgut.h" +#include "thread.h" /* Logger parameters */ @@ -69,6 +69,9 @@ static bool exit_hook_registered = false; static bool loggin_in_progress = false; static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; +#ifdef WIN32 +static long mutex_initlock = 0; +#endif void init_logger(const char *root_path) @@ -138,11 +141,10 @@ exit_if_necessary(int elevel) } /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) #ifdef WIN32 - if (main_tid != GetCurrentThreadId()) ExitThread(elevel); #else - if (!pthread_equal(main_tid, pthread_self())) pthread_exit(NULL); #endif else @@ -174,6 +176,16 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) /* * There is no need to lock if this is elog() from upper elog(). */ +#ifdef WIN32 + if (log_file_mutex == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (log_file_mutex == NULL) + pthread_mutex_init(&log_file_mutex, NULL); + InterlockedExchange(&mutex_initlock, 0); + } +#endif pthread_mutex_lock(&log_file_mutex); loggin_in_progress = true; @@ -550,7 +562,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, "r"); + control_file = fopen(control, PG_BINARY_R); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -596,7 +608,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, "w"); + control_file = fopen(control, PG_BINARY_W); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 56263cfa..1a7aae0b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -42,12 +42,6 @@ static char *password = NULL; bool prompt_password = true; bool force_password = false; -#ifdef WIN32 -DWORD main_tid = 0; -#else -pthread_t main_tid = 0; -#endif - /* Database connections */ static PGcancel *volatile cancel_conn = NULL; @@ -995,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) + if (!isprint(opts[i].val)) //opts[i].val > 128 || continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1052,6 +1046,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; + len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1059,6 +1054,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) optstring = longopts_to_optstring(longopts, len); /* Assign named options */ + optind = 2; while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); @@ -1225,7 +1221,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "# \n\r\t\v"); + i = j = strcspn(s, "#\n\r\t\v");//removed space memcpy(dst, s, j); } diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 803d2c57..a27cea83 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -15,7 +15,6 @@ #include "pqexpbuffer.h" #include -#include #include #include "logger.h" @@ -94,13 +93,6 @@ extern const char *PROGRAM_VERSION; extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; -/* ID of the main thread */ -#ifdef WIN32 -extern DWORD main_tid; -#else -extern pthread_t main_tid; -#endif - extern void pgut_help(bool details); /* diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 00000000..da3544a1 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 00000000..0a113d58 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,32 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c index afc7f07d..cb693268 100644 --- a/src/validate.c +++ b/src/validate.c @@ -11,10 +11,11 @@ #include "pg_probackup.h" #include -#include #include -static void pgBackupValidateFiles(void *arg); +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; @@ -42,8 +43,9 @@ pgBackupValidate(pgBackup *backup) parray *files; bool corrupted = false; bool validation_isok = true; - pthread_t validate_threads[num_threads]; - validate_files_args *validate_threads_args[num_threads]; + /* arrays with meta info for multi threaded validate */ + pthread_t *validate_threads; + validate_files_args *validate_threads_args; int i; /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ @@ -77,36 +79,44 @@ pgBackupValidate(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } + /* init thread args with own file lists */ + validate_threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + validate_threads_args = (validate_files_args *) + palloc(sizeof(validate_files_args) * num_threads); + /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = pg_malloc(sizeof(validate_files_args)); + validate_files_args *arg = &(validate_threads_args[i]); + arg->files = files; arg->corrupted = false; /* By default there are some error */ arg->ret = 1; - validate_threads_args[i] = arg; - pthread_create(&validate_threads[i], NULL, - (void *(*)(void *)) pgBackupValidateFiles, arg); + pthread_create(&validate_threads[i], NULL, pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { + validate_files_args *arg = &(validate_threads_args[i]); + pthread_join(validate_threads[i], NULL); - if (validate_threads_args[i]->corrupted) + if (arg->corrupted) corrupted = true; - if (validate_threads_args[i]->ret == 1) + if (arg->ret == 1) validation_isok = false; - pg_free(validate_threads_args[i]); } if (!validation_isok) elog(ERROR, "Data files validation failed"); + pfree(validate_threads); + pfree(validate_threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); @@ -127,10 +137,10 @@ pgBackupValidate(pgBackup *backup) * rather throw a WARNING and set arguments->corrupted = true. * This is necessary to update backup status. */ -static void +static void * pgBackupValidateFiles(void *arg) { - int i; + int i; validate_files_args *arguments = (validate_files_args *)arg; pg_crc32 crc; @@ -139,7 +149,7 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) @@ -198,6 +208,8 @@ pgBackupValidateFiles(void *arg) /* Data files validation is successful */ arguments->ret = 0; + + return NULL; } /* @@ -267,7 +279,7 @@ do_validate_all(void) static void do_validate_instance(void) { - char *current_backup_id; + char *current_backup_id; int i; parray *backups; pgBackup *current_backup = NULL; diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From bd1f5c478c0c0bbd554f86fd234051c3a0eb18e3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Fri, 8 Jun 2018 13:22:39 +0300 Subject: [PATCH 38/66] Some changes after Windows testing --- doit.cmd | 2 +- msvs/template.pg_probackup.vcxproj | 4 ++++ msvs/template.pg_probackup96.vcxproj | 4 ++++ src/utils/thread.c | 1 + src/utils/thread.h | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doit.cmd b/doit.cmd index cc29bc88..b46e3b36 100644 --- a/doit.cmd +++ b/doit.cmd @@ -1 +1 @@ -perl win32build.pl "C:\Program Files\PostgresProEnterprise\10" "C:\projects\pgwininstall2\pgwininstall\builddir\postgresql\postgresql-10.3\src" \ No newline at end of file +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 044a3e73..9b1d84dd 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -191,6 +191,8 @@ + + @@ -202,6 +204,8 @@ + + diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj index c095d7cc..185c6382 100644 --- a/msvs/template.pg_probackup96.vcxproj +++ b/msvs/template.pg_probackup96.vcxproj @@ -190,6 +190,8 @@ + + @@ -200,6 +202,8 @@ + + diff --git a/src/utils/thread.c b/src/utils/thread.c index da3544a1..a12a1c83 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -12,6 +12,7 @@ pthread_t main_tid = 0; #ifdef WIN32 +#include typedef struct win32_pthread { diff --git a/src/utils/thread.h b/src/utils/thread.h index 0a113d58..1a50f1da 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -11,6 +11,7 @@ #define PROBACKUP_THREAD_H #ifdef WIN32 +#include "postgres_fe.h" #include "port/pthread-win32.h" /* Use native win32 threads on Windows */ From 74d9e437749f0fb3f99aab3483e499f2d92bf05b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 12:32:40 +0300 Subject: [PATCH 39/66] PGPRO-1646: Use int for write_size instead of size_t --- src/backup.c | 33 +++++++++++++++------------------ src/data.c | 8 ++++---- src/dir.c | 8 ++++---- src/pg_probackup.h | 4 ++-- src/restore.c | 4 ++-- src/validate.c | 5 ++--- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/backup.c b/src/backup.c index f24a0b0f..c96d8f17 100644 --- a/src/backup.c +++ b/src/backup.c @@ -357,7 +357,7 @@ remote_copy_file(PGconn *conn, pgFile* file) elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); } - file->write_size = file->read_size; + file->write_size = (int) file->read_size; FIN_CRC32C(file->crc); fclose(out); @@ -438,8 +438,8 @@ remote_backup_files(void *arg) /* receive the data from stream and write to backup file */ remote_copy_file(file_backup_conn, file); - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied %d bytes", + file->path, file->write_size); PQfinish(file_backup_conn); } @@ -2099,27 +2099,24 @@ backup_files(void *arg) continue; } } - else - /* TODO: - * Check if file exists in previous backup - * If exists: - * if mtime > start_backup_time of parent backup, - * copy file to backup - * if mtime < start_backup_time - * calculate crc, compare crc to old file - * if crc is the same -> skip file - */ - if (!copy_file(arguments->from_root, - arguments->to_root, - file)) + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; } - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied %d bytes", + file->path, file->write_size); } else elog(LOG, "unexpected file type %d", buf.st_mode); diff --git a/src/data.c b/src/data.c index be07ce64..4605b275 100644 --- a/src/data.c +++ b/src/data.c @@ -229,8 +229,8 @@ backup_data_page(backup_files_args *arguments, BackupMode backup_mode) { BackupPageHeader header; - Page page = malloc(BLCKSZ); - Page compressed_page = NULL; + Page page = malloc(BLCKSZ); + Page compressed_page = NULL; XLogRecPtr page_lsn = 0; size_t write_buffer_size; char write_buffer[BLCKSZ+sizeof(header)]; @@ -385,7 +385,7 @@ backup_data_page(backup_files_args *arguments, /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", @@ -843,7 +843,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->read_size += read_len; } - file->write_size = file->read_size; + file->write_size = (int) file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index 20725f88..6cbd5f2e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -821,11 +821,11 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," + fprintf(out, "{\"path\":\"%s\", \"size\":\"%d\",\"mode\":\"%u\"," "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," "\"compress_alg\":\"%s\"", - path, (unsigned long) file->write_size, file->mode, - file->is_datafile?1:0, file->is_cfs?1:0, file->crc, + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg)); if (file->is_datafile) @@ -1032,7 +1032,7 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); - file->write_size = (size_t) write_size; + file->write_size = (int) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8face84d..15c29204 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,7 +88,7 @@ typedef struct pgFile size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - size_t write_size; /* size of the backed-up file. BYTES_INVALID means + int write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ pg_crc32 crc; /* CRC value of the file, regular file only */ @@ -160,7 +160,7 @@ typedef enum ShowFormat /* special values of pgBackup fields */ -#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) typedef struct pgBackupConfig diff --git a/src/restore.c b/src/restore.c index 0e42c4c4..3bf87d7c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -777,8 +777,8 @@ restore_files(void *arg) /* print size of restored file */ if (file->write_size != BYTES_INVALID) - elog(LOG, "Restored file %s : %lu bytes", - file->path, (unsigned long) file->write_size); + elog(LOG, "Restored file %s : %d bytes", + file->path, file->write_size); } /* Data files restoring is successful */ diff --git a/src/validate.c b/src/validate.c index cb693268..97a248ec 100644 --- a/src/validate.c +++ b/src/validate.c @@ -189,9 +189,8 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : %lu. Expected %lu", - file->path, (unsigned long) file->write_size, - (unsigned long) st.st_size); + elog(WARNING, "Invalid size of backup file \"%s\" : %d. Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); arguments->corrupted = true; break; } From 1da749aa5ea6a1e65b47ebed7e5e2f56bf725a76 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 13:55:30 +0300 Subject: [PATCH 40/66] Default compression level is 1, use NOT_DEFINED_COMPRESS for not defined compression algorithm --- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index c66179e1..668744b2 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -564,7 +564,7 @@ compress_init(void) if (backup_subcmd != SET_CONFIG) { if (compress_level != DEFAULT_COMPRESS_LEVEL - && compress_alg == NONE_COMPRESS) + && compress_alg == NOT_DEFINED_COMPRESS) elog(ERROR, "Cannot specify compress-level option without compress-alg option"); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 15c29204..b01094e3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -353,7 +353,7 @@ extern CompressAlg compress_alg; extern int compress_level; extern bool compress_shortcut; -#define DEFAULT_COMPRESS_LEVEL 6 +#define DEFAULT_COMPRESS_LEVEL 1 extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); From 03a3fb8a14558698e4646ea14acaf44f15697b06 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 15:14:44 +0300 Subject: [PATCH 41/66] PGPRO-1646: Use int64 for write_size --- src/backup.c | 6 +++--- src/data.c | 2 +- src/dir.c | 7 ++++--- src/pg_probackup.h | 3 ++- src/restore.c | 2 +- src/validate.c | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index c96d8f17..e58da8b8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -357,7 +357,7 @@ remote_copy_file(PGconn *conn, pgFile* file) elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); } - file->write_size = (int) file->read_size; + file->write_size = (int64) file->read_size; FIN_CRC32C(file->crc); fclose(out); @@ -438,7 +438,7 @@ remote_backup_files(void *arg) /* receive the data from stream and write to backup file */ remote_copy_file(file_backup_conn, file); - elog(VERBOSE, "File \"%s\". Copied %d bytes", + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", file->path, file->write_size); PQfinish(file_backup_conn); } @@ -2115,7 +2115,7 @@ backup_files(void *arg) continue; } - elog(VERBOSE, "File \"%s\". Copied %d bytes", + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", file->path, file->write_size); } else diff --git a/src/data.c b/src/data.c index 4605b275..4ab8a1c0 100644 --- a/src/data.c +++ b/src/data.c @@ -843,7 +843,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->read_size += read_len; } - file->write_size = (int) file->read_size; + file->write_size = (int64) file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index 6cbd5f2e..64058b62 100644 --- a/src/dir.c +++ b/src/dir.c @@ -821,8 +821,9 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"%d\",\"mode\":\"%u\"," - "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, @@ -1032,7 +1033,7 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); - file->write_size = (int) write_size; + file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b01094e3..b4ebfb98 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,9 +88,10 @@ typedef struct pgFile size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - int write_size; /* size of the backed-up file. BYTES_INVALID means + int64 write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ + /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ diff --git a/src/restore.c b/src/restore.c index 3bf87d7c..a35d50b2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -777,7 +777,7 @@ restore_files(void *arg) /* print size of restored file */ if (file->write_size != BYTES_INVALID) - elog(LOG, "Restored file %s : %d bytes", + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", file->path, file->write_size); } diff --git a/src/validate.c b/src/validate.c index 97a248ec..41467f25 100644 --- a/src/validate.c +++ b/src/validate.c @@ -189,7 +189,7 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : %d. Expected %lu", + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", file->path, file->write_size, (unsigned long) st.st_size); arguments->corrupted = true; break; From 1304c88ce74e0a95736f667b7bd107382526320e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 15:45:03 +0300 Subject: [PATCH 42/66] Minor style fixes --- src/backup.c | 12 ++++++------ src/configure.c | 3 +-- src/dir.c | 10 +++------- src/restore.c | 2 +- src/utils/logger.c | 4 ++-- src/utils/pgut.c | 5 ++--- 6 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index e58da8b8..e8efd4fb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -105,7 +105,7 @@ static void write_backup_file_list(parray *files, const char *root); static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); -static void StreamLog(void *arg); +static void *StreamLog(void *arg); static void get_remote_pgdata_filelist(parray *files); static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); @@ -599,8 +599,7 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; - pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, - &stream_thread_arg); + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } /* initialize backup list */ @@ -1051,8 +1050,7 @@ confirm_block_size(const char *name, int blcksz) "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); - PQclear(res);//bad pointer to endp - + PQclear(res); } /* @@ -2539,7 +2537,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* * Start the log streaming */ -static void +static void * StreamLog(void *arg) { XLogRecPtr startpos; @@ -2613,6 +2611,8 @@ StreamLog(void *arg) PQfinish(stream_arg->conn); stream_arg->conn = NULL; + + return NULL; } /* diff --git a/src/configure.c b/src/configure.c index 2fc27892..11c068a5 100644 --- a/src/configure.c +++ b/src/configure.c @@ -132,8 +132,7 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - //fprintf(out, "system-identifier = %li\n", config->system_identifier); - fprintf(out, "system-identifier = %" INT64_MODIFIER "u\n", config->system_identifier); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) diff --git a/src/dir.c b/src/dir.c index 64058b62..c29e4bbc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -100,12 +100,9 @@ static void dir_list_file_internal(parray *files, const char *root, int dir_create_dir(const char *dir, mode_t mode) { - char copy[MAXPGPATH]; - char *parent; + char parent[MAXPGPATH]; - strncpy(copy, dir, MAXPGPATH); - - parent = pstrdup(dir); + strncpy(parent, dir, MAXPGPATH); get_parent_directory(parent); /* Create parent first */ @@ -120,7 +117,6 @@ dir_create_dir(const char *dir, mode_t mode) elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); } - pfree(parent); return 0; } @@ -155,7 +151,7 @@ pgFileInit(const char *path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); - file->name = 0; + file->name = NULL; file->size = 0; file->mode = 0; diff --git a/src/restore.c b/src/restore.c index a35d50b2..dc339fd5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -92,7 +92,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, int dest_backup_index = 0; int base_full_backup_index = 0; int corrupted_backup_index = 0; - char *action = is_restore ? "Restore":"Validate"; + char *action = is_restore ? "Restore":"Validate"; if (is_restore) { diff --git a/src/utils/logger.c b/src/utils/logger.c index 8fd78739..7a642330 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -562,7 +562,7 @@ open_logfile(FILE **file, const char *filename_format) { char buf[1024]; - control_file = fopen(control, PG_BINARY_R); + control_file = fopen(control, "r"); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); @@ -608,7 +608,7 @@ logfile_open: { time_t timestamp = time(NULL); - control_file = fopen(control, PG_BINARY_W); + control_file = fopen(control, "w"); if (control_file == NULL) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 1a7aae0b..48f1496b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -989,7 +989,7 @@ longopts_to_optstring(const struct option opts[], const size_t len) s = result; for (i = 0; i < len; i++) { - if (!isprint(opts[i].val)) //opts[i].val > 128 || + if (!isprint(opts[i].val)) continue; *s++ = opts[i].val; if (opts[i].has_arg != no_argument) @@ -1046,7 +1046,6 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) struct option *longopts; size_t len; - len = option_length(options); longopts = pgut_newarray(struct option, len + 1); option_copy(longopts, options, len); @@ -1221,7 +1220,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "#\n\r\t\v");//removed space + i = j = strcspn(s, "#\n\r\t\v"); memcpy(dst, s, j); } From 0afc00439ce09c77fd312f3656de4bbd2b9a3d90 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 18:05:18 +0300 Subject: [PATCH 43/66] Use optind to parse subcommand name --- src/pg_probackup.c | 222 ++++++++++++++++++++++----------------------- src/pg_probackup.h | 27 +++--- src/utils/pgut.c | 9 +- 3 files changed, 128 insertions(+), 130 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 668744b2..9a61c5d0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -64,7 +64,7 @@ static char *target_inclusive; static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; -static char *target_action = NULL;; +static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; @@ -99,9 +99,10 @@ ShowFormat show_format = SHOW_PLAIN; /* current settings */ pgBackup current; -ProbackupSubcmd backup_subcmd; +ProbackupSubcmd backup_subcmd = NO_CMD; -bool help = false; +static bool help_opt = false; +static bool version_opt = false; static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); @@ -114,7 +115,8 @@ static void compress_init(void); static pgut_option options[] = { /* directory options */ - { 'b', 1, "help", &help, SOURCE_CMDLINE }, + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 'b', 'V', "version", &version_opt, SOURCE_CMDLINE }, { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, /* common options */ @@ -152,7 +154,7 @@ static pgut_option options[] = { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, /* TODO not implemented yet */ - { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, /* retention options */ { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, @@ -198,6 +200,9 @@ main(int argc, char *argv[]) /* Check if backup_path is directory. */ struct stat stat_buf; int rc; + int i, + len = 0, + allocated = 0; /* initialize configuration */ pgBackup_init(¤t); @@ -210,106 +215,94 @@ main(int argc, char *argv[]) */ main_tid = pthread_self(); - /* Parse subcommands and non-subcommand options */ - if (argc > 1) - { - if (strcmp(argv[1], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH; - else if (strcmp(argv[1], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET; - else if (strcmp(argv[1], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE; - else if (strcmp(argv[1], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE; - else if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE_SUBCMD; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG; - else if (strcmp(argv[1], "--help") == 0 - || strcmp(argv[1], "help") == 0 - || strcmp(argv[1], "-?") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) - { - if (argc == 2) - { -#ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); -#else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); -#endif - exit(0); - } - else if (strcmp(argv[2], "--help") == 0) - help_command(argv[1]); - else - elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); - } - else - elog(ERROR, "Unknown subcommand"); - } - /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ - if (backup_subcmd == BACKUP || - backup_subcmd == RESTORE || - backup_subcmd == VALIDATE || - backup_subcmd == DELETE_SUBCMD) + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) { - int i, - len = 0, - allocated = 0; + int arglen = strlen(argv[i]); - allocated = sizeof(char) * MAXPGPATH; - command = (char *) palloc(allocated); - - for (i = 0; i < argc; i++) + if (arglen + len > allocated) { - int arglen = strlen(argv[i]); - - if (arglen + len > allocated) - { - allocated *= 2; - command = repalloc(command, allocated); - } - - strncpy(command + len, argv[i], arglen); - len += arglen; - command[len++] = ' '; + allocated *= 2; + command = repalloc(command, allocated); } - command[len] = '\0'; + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; } + command[len] = '\0'; + /* Parse command line arguments */ pgut_getopt(argc, argv, options); - if (help) - help_command(argv[2]); + /* Process a command */ + if (optind < argc) + { + if (strcmp(argv[optind], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[optind], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[optind], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[optind], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[optind], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[optind], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[optind], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[optind], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[optind], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[optind], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[optind], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[optind], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[optind], "help") == 0) + { + if (argc - optind < 2) + help_pg_probackup(); + else + help_command(argv[optind + 1]); + } + else if (strcmp(argv[optind], "version") == 0) + version_opt = true; + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); + } + + if (help_opt) + { + if (backup_subcmd == NO_CMD || argc - optind < 1) + help_pg_probackup(); + else + help_command(argv[optind]); + } + else if (version_opt) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + return 0; + } + else if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); /* backup_path is required for all pg_probackup commands except help */ if (backup_path == NULL) @@ -343,7 +336,7 @@ main(int argc, char *argv[]) } /* Option --instance is required for all commands except init and show */ - if (backup_subcmd != INIT && backup_subcmd != SHOW && backup_subcmd != VALIDATE) + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) { if (instance_name == NULL) elog(ERROR, "required parameter not specified: --instance"); @@ -363,7 +356,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT && backup_subcmd != ADD_INSTANCE) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -375,7 +368,7 @@ main(int argc, char *argv[]) * Read options from env variables or from config file, * unless we're going to set them via set-config. */ - if (instance_name && backup_subcmd != SET_CONFIG) + if (instance_name && backup_subcmd != SET_CONFIG_CMD) { /* Read environment variables */ pgut_getopt_env(options); @@ -398,10 +391,10 @@ main(int argc, char *argv[]) /* Sanity check of --backup-id option */ if (backup_id_string_param != NULL) { - if (backup_subcmd != RESTORE - && backup_subcmd != VALIDATE - && backup_subcmd != DELETE_SUBCMD - && backup_subcmd != SHOW) + if (backup_subcmd != RESTORE_CMD + && backup_subcmd != VALIDATE_CMD + && backup_subcmd != DELETE_CMD + && backup_subcmd != SHOW_CMD) elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", argv[1]); @@ -429,7 +422,7 @@ main(int argc, char *argv[]) pgdata_exclude_dir[i] = "pg_log"; } - if (backup_subcmd == VALIDATE || backup_subcmd == RESTORE) + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, @@ -445,17 +438,17 @@ main(int argc, char *argv[]) /* do actual operation */ switch (backup_subcmd) { - case ARCHIVE_PUSH: + case ARCHIVE_PUSH_CMD: return do_archive_push(wal_file_path, wal_file_name, file_overwrite); - case ARCHIVE_GET: + case ARCHIVE_GET_CMD: return do_archive_get(wal_file_path, wal_file_name); - case ADD_INSTANCE: + case ADD_INSTANCE_CMD: return do_add_instance(); - case DELETE_INSTANCE: + case DELETE_INSTANCE_CMD: return do_delete_instance(); - case INIT: + case INIT_CMD: return do_init(); - case BACKUP: + case BACKUP_CMD: { const char *backup_mode; time_t start_time; @@ -470,20 +463,20 @@ main(int argc, char *argv[]) return do_backup(start_time); } - case RESTORE: + case RESTORE_CMD: return do_restore_or_validate(current.backup_id, recovery_target_options, true); - case VALIDATE: + case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0) return do_validate_all(); else return do_restore_or_validate(current.backup_id, recovery_target_options, false); - case SHOW: + case SHOW_CMD: return do_show(current.backup_id); - case DELETE_SUBCMD: + case DELETE_CMD: if (delete_expired && backup_id_string_param) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); if (!delete_expired && !delete_wal && !backup_id_string_param) @@ -494,10 +487,13 @@ main(int argc, char *argv[]) return do_retention_purge(); else return do_delete(current.backup_id); - case SHOW_CONFIG: + case SHOW_CONFIG_CMD: return do_configure(true); - case SET_CONFIG: + case SET_CONFIG_CMD: return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); } return 0; @@ -561,7 +557,7 @@ compress_init(void) if (compress_shortcut) compress_alg = ZLIB_COMPRESS; - if (backup_subcmd != SET_CONFIG) + if (backup_subcmd != SET_CONFIG_CMD) { if (compress_level != DEFAULT_COMPRESS_LEVEL && compress_alg == NOT_DEFINED_COMPRESS) @@ -574,7 +570,7 @@ compress_init(void) if (compress_level == 0) compress_alg = NOT_DEFINED_COMPRESS; - if (backup_subcmd == BACKUP || backup_subcmd == ARCHIVE_PUSH) + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ if (compress_alg == ZLIB_COMPRESS) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b4ebfb98..39e2db67 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -139,18 +139,19 @@ typedef enum BackupMode typedef enum ProbackupSubcmd { - INIT = 0, - ARCHIVE_PUSH, - ARCHIVE_GET, - ADD_INSTANCE, - DELETE_INSTANCE, - BACKUP, - RESTORE, - VALIDATE, - SHOW, - DELETE_SUBCMD, - SET_CONFIG, - SHOW_CONFIG + NO_CMD = 0, + INIT_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + SHOW_CMD, + DELETE_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD } ProbackupSubcmd; typedef enum ShowFormat @@ -367,7 +368,7 @@ extern ShowFormat show_format; /* current settings */ extern pgBackup current; -extern ProbackupSubcmd backup_subcmd; +extern ProbackupSubcmd backup_subcmd; /* in dir.c */ /* exclude directory list for $PGDATA file listing */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 48f1496b..f1b5babd 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -135,8 +135,10 @@ static const unit_conversion time_unit_conversion_table[] = static size_t option_length(const pgut_option opts[]) { - size_t len; + size_t len; + for (len = 0; opts && opts[len].type; len++) { } + return len; } @@ -156,7 +158,7 @@ option_has_arg(char type) static void option_copy(struct option dst[], const pgut_option opts[], size_t len) { - size_t i; + size_t i; for (i = 0; i < len; i++) { @@ -1047,13 +1049,12 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) size_t len; len = option_length(options); - longopts = pgut_newarray(struct option, len + 1); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); option_copy(longopts, options, len); optstring = longopts_to_optstring(longopts, len); /* Assign named options */ - optind = 2; while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { opt = option_find(c, options); From d3b1127ab3c4671391fdf50747175fc5370dd2c3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 9 Jun 2018 18:14:47 +0300 Subject: [PATCH 44/66] Remove duplicate help command code --- src/pg_probackup.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9a61c5d0..a6fd22a3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -271,10 +271,8 @@ main(int argc, char *argv[]) backup_subcmd = SHOW_CONFIG_CMD; else if (strcmp(argv[optind], "help") == 0) { - if (argc - optind < 2) - help_pg_probackup(); - else - help_command(argv[optind + 1]); + optind++; + help_opt = true; } else if (strcmp(argv[optind], "version") == 0) version_opt = true; @@ -284,7 +282,7 @@ main(int argc, char *argv[]) if (help_opt) { - if (backup_subcmd == NO_CMD || argc - optind < 1) + if (argc - optind < 1) help_pg_probackup(); else help_command(argv[optind]); From f336db64820ee751f67a34824c4fc68e508c70ad Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 12:01:18 +0300 Subject: [PATCH 45/66] Add include pg_getopt.h --- src/pg_probackup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a6fd22a3..1e4635e2 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,6 +17,7 @@ #include #include #include +#include "pg_getopt.h" const char *PROGRAM_VERSION = "2.0.17"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; From 0e14954483b46bbec15c5f65f4f0e94e2a5ffb18 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 12:32:06 +0300 Subject: [PATCH 46/66] Update windows project templates --- msvs/template.pg_probackup.vcxproj | 5 ++--- msvs/template.pg_probackup96.vcxproj | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index 9b1d84dd..46a7b2c2 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -163,7 +163,6 @@ libc;%(IgnoreSpecificDefaultLibraries) - @@ -204,8 +203,8 @@ - - + + diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj index 185c6382..46e019ba 100644 --- a/msvs/template.pg_probackup96.vcxproj +++ b/msvs/template.pg_probackup96.vcxproj @@ -163,7 +163,6 @@ libc;%(IgnoreSpecificDefaultLibraries) - @@ -202,8 +201,8 @@ - - + + From b10fdf119de01ae07e8d4235e87931059751e2d4 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 14 Jun 2018 16:22:25 +0300 Subject: [PATCH 47/66] Update command line parse --- src/pg_probackup.c | 73 +++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1e4635e2..75ae0314 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -241,44 +241,49 @@ main(int argc, char *argv[]) command[len] = '\0'; /* Parse command line arguments */ - pgut_getopt(argc, argv, options); - - /* Process a command */ - if (optind < argc) + optind = 1; + /* process command-line options */ + while (optind < argc) { - if (strcmp(argv[optind], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[optind], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[optind], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[optind], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[optind], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[optind], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[optind], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[optind], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[optind], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[optind], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[optind], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[optind], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[optind], "help") == 0) + pgut_getopt(argc, argv, options); + /* Process a command */ + if (optind < argc) { + if (strcmp(argv[optind], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[optind], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[optind], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[optind], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[optind], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[optind], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[optind], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[optind], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[optind], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[optind], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[optind], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[optind], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[optind], "help") == 0) + { + optind++; + help_opt = true; + } + else if (strcmp(argv[optind], "version") == 0) + version_opt = true; + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); optind++; - help_opt = true; } - else if (strcmp(argv[optind], "version") == 0) - version_opt = true; - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); } if (help_opt) From dac3958eecadc15d9529602c0f57c3e5a6f77100 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Jun 2018 19:42:19 +0300 Subject: [PATCH 48/66] Return command line parse algorithm --- src/help.c | 2 +- src/pg_probackup.c | 169 +++++++++++++++++++++++---------------------- 2 files changed, 87 insertions(+), 84 deletions(-) diff --git a/src/help.c b/src/help.c index 28500e75..1d147989 100644 --- a/src/help.c +++ b/src/help.c @@ -56,7 +56,7 @@ help_command(char *command) || strcmp(command, "-V") == 0) printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); else - printf(_("Unknown command. Try pg_probackup help\n")); + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); exit(0); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 75ae0314..e0cb5479 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -103,7 +103,6 @@ pgBackup current; ProbackupSubcmd backup_subcmd = NO_CMD; static bool help_opt = false; -static bool version_opt = false; static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); @@ -117,7 +116,6 @@ static pgut_option options[] = { /* directory options */ { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, - { 'b', 'V', "version", &version_opt, SOURCE_CMDLINE }, { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, /* common options */ @@ -196,14 +194,12 @@ static pgut_option options[] = int main(int argc, char *argv[]) { - char *command = NULL; + char *command = NULL, + *command_name; char path[MAXPGPATH]; /* Check if backup_path is directory. */ struct stat stat_buf; int rc; - int i, - len = 0, - allocated = 0; /* initialize configuration */ pgBackup_init(¤t); @@ -216,97 +212,104 @@ main(int argc, char *argv[]) */ main_tid = pthread_self(); + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ - allocated = sizeof(char) * MAXPGPATH; - command = (char *) palloc(allocated); - - for (i = 0; i < argc; i++) + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD) { - int arglen = strlen(argv[i]); + int i, + len = 0, + allocated = 0; - if (arglen + len > allocated) + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) { - allocated *= 2; - command = repalloc(command, allocated); - } + int arglen = strlen(argv[i]); - strncpy(command + len, argv[i], arglen); - len += arglen; - command[len++] = ' '; - } - - command[len] = '\0'; - - /* Parse command line arguments */ - optind = 1; - /* process command-line options */ - while (optind < argc) - { - pgut_getopt(argc, argv, options); - /* Process a command */ - if (optind < argc) - { - if (strcmp(argv[optind], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[optind], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[optind], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[optind], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[optind], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[optind], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[optind], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[optind], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[optind], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[optind], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[optind], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[optind], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[optind], "help") == 0) + if (arglen + len > allocated) { - optind++; - help_opt = true; + allocated *= 2; + command = repalloc(command, allocated); } - else if (strcmp(argv[optind], "version") == 0) - version_opt = true; - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[optind]); - optind++; + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; } + + command[len] = '\0'; } + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + if (help_opt) - { - if (argc - optind < 1) - help_pg_probackup(); - else - help_command(argv[optind]); - } - else if (version_opt) - { -#ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); -#else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); -#endif - return 0; - } - else if (backup_subcmd == NO_CMD) - elog(ERROR, "No subcommand specified"); + help_command(command_name); /* backup_path is required for all pg_probackup commands except help */ if (backup_path == NULL) From bc016cfa61ec3f42060f9626785c7df1130feafb Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 19 Jun 2018 13:52:31 +0300 Subject: [PATCH 49/66] some change for removing visual studio build error: potentially uninitialized local pointer variable 'error_args' --- src/utils/logger.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 7a642330..fe69a9d8 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -190,14 +190,14 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) loggin_in_progress = true; /* We need copy args only if we need write to error log file */ - if (write_to_error_log) - va_copy(error_args, args); + //if (write_to_error_log) + // va_copy(error_args, args); /* * We need copy args only if we need write to stderr. But do not copy args * if we need to log only to stderr. */ - if (write_to_stderr && write_to_file) - va_copy(std_args, args); +// if (write_to_stderr && write_to_file) +// va_copy(std_args, args); if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", @@ -238,7 +238,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - + + va_copy(error_args, args); vfprintf(error_log_file, fmt, error_args); fputc('\n', error_log_file); fflush(error_log_file); @@ -253,7 +254,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_stderr) { write_elevel(stderr, elevel); - + va_copy(std_args, args); if (write_to_file) vfprintf(stderr, fmt, std_args); else From a2959609c8d692d8caca0050c7d85952f16161a8 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 19 Jun 2018 14:50:35 +0300 Subject: [PATCH 50/66] Some changes for windows --- src/utils/logger.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index fe69a9d8..22fc0508 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -177,6 +177,9 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) * There is no need to lock if this is elog() from upper elog(). */ #ifdef WIN32 + std_args = NULL; + error_args = NULL; + if (log_file_mutex == NULL) { while (InterlockedExchange(&mutex_initlock, 1) == 1) @@ -190,14 +193,14 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) loggin_in_progress = true; /* We need copy args only if we need write to error log file */ - //if (write_to_error_log) - // va_copy(error_args, args); + if (write_to_error_log) + va_copy(error_args, args); /* * We need copy args only if we need write to stderr. But do not copy args * if we need to log only to stderr. */ -// if (write_to_stderr && write_to_file) -// va_copy(std_args, args); + if (write_to_stderr && write_to_file) + va_copy(std_args, args); if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", @@ -239,7 +242,6 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - va_copy(error_args, args); vfprintf(error_log_file, fmt, error_args); fputc('\n', error_log_file); fflush(error_log_file); @@ -254,7 +256,6 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_stderr) { write_elevel(stderr, elevel); - va_copy(std_args, args); if (write_to_file) vfprintf(stderr, fmt, std_args); else From 6b539ad59a15367850cd65a85d42ee37a9b9fe36 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 28 Jun 2018 12:33:16 +0300 Subject: [PATCH 51/66] set 0 to PageBitmapIsAbsent --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 39e2db67..55a00636 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -112,7 +112,7 @@ typedef struct pgFile /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ -#define PageBitmapIsAbsent -1 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ +#define PageBitmapIsAbsent 0 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ /* Current state of backup */ typedef enum BackupStatus From 7f87c1cebebd0ab1a3a100d198a6940ee37f550c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 29 Jun 2018 14:01:08 +0300 Subject: [PATCH 52/66] Add pagemap_isabsent instead of PageBitmapIsAbsent define --- src/backup.c | 8 ++++---- src/data.c | 7 +++---- src/dir.c | 3 ++- src/pg_probackup.h | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index e8efd4fb..420bcb78 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2403,12 +2403,12 @@ make_pagemap_from_ptrack(parray *files) if (file->is_datafile) { - if (file->tblspcOid == tblspcOid_with_ptrack_init - && file->dbOid == dbOid_with_ptrack_init) + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) { /* ignore ptrack if ptrack_init exists */ elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; continue; } @@ -2461,7 +2461,7 @@ make_pagemap_from_ptrack(parray *files) * - target relation was deleted. */ elog(VERBOSE, "Ptrack is missing for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; } } } diff --git a/src/data.c b/src/data.c index 4ab8a1c0..fd09e5e7 100644 --- a/src/data.c +++ b/src/data.c @@ -430,7 +430,7 @@ backup_data_file(backup_files_args* arguments, if ((backup_mode == BACKUP_MODE_DIFF_PAGE || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev) + file->exists_in_prev && !file->pagemap_isabsent) { /* * There are no changed blocks since last backup. We want make @@ -494,9 +494,8 @@ backup_data_file(backup_files_args* arguments, * If page map is empty or file is not present in previous backup * backup all pages of the relation. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty - || file->pagemap.bitmapsize == PageBitmapIsAbsent - || !file->exists_in_prev) + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) { for (blknum = 0; blknum < nblocks; blknum++) { diff --git a/src/dir.c b/src/dir.c index c29e4bbc..0375a4de 100644 --- a/src/dir.c +++ b/src/dir.c @@ -161,7 +161,8 @@ pgFileInit(const char *path) file->is_datafile = false; file->linked = NULL; file->pagemap.bitmap = NULL; - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; file->tblspcOid = 0; file->dbOid = 0; file->relOid = 0; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 55a00636..75788cd7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -108,11 +108,12 @@ typedef struct pgFile CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ } pgFile; /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ -#define PageBitmapIsAbsent 0 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ /* Current state of backup */ typedef enum BackupStatus From 308022845329bb8e5318b390ad13877a510414d5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 4 Jul 2018 18:37:36 +0300 Subject: [PATCH 53/66] fix: add restore-target-name to validate --help. Submitted by Ludmila Mantrova. --- src/help.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/help.c b/src/help.c index 1d147989..b0225575 100644 --- a/src/help.c +++ b/src/help.c @@ -125,6 +125,7 @@ help_pg_probackup(void) printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); @@ -332,6 +333,8 @@ help_validate(void) printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From be8f3be6cd7b11e75c4a0e95a3703ba625b2b766 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 5 Jul 2018 13:19:18 +0300 Subject: [PATCH 54/66] PGPRO-1759: Fix messages and variable names, it should be data not date --- src/catalog.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index f19628c3..a0a8e5dc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -250,14 +250,14 @@ IsDir(const char *dirpath, const char *entry) parray * catalog_get_backup_list(time_t requested_backup_id) { - DIR *date_dir = NULL; - struct dirent *date_ent = NULL; + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; parray *backups = NULL; pgBackup *backup = NULL; /* open backup instance backups directory */ - date_dir = opendir(backup_instance_path); - if (date_dir == NULL) + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) { elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, strerror(errno)); @@ -266,21 +266,21 @@ catalog_get_backup_list(time_t requested_backup_id) /* scan the directory and list backups */ backups = parray_new(); - for (; (date_ent = readdir(date_dir)) != NULL; errno = 0) + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) { char backup_conf_path[MAXPGPATH]; - char date_path[MAXPGPATH]; + char data_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, date_ent->d_name) - || date_ent->d_name[0] == '.') + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(date_path, backup_instance_path, date_ent->d_name); + join_path_components(data_path, backup_instance_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ - snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); /* ignore corrupted backups */ @@ -298,8 +298,8 @@ catalog_get_backup_list(time_t requested_backup_id) if (errno && errno != ENOENT) { - elog(WARNING, "cannot read date directory \"%s\": %s", - date_ent->d_name, strerror(errno)); + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); goto err_proc; } } @@ -310,16 +310,16 @@ catalog_get_backup_list(time_t requested_backup_id) goto err_proc; } - closedir(date_dir); - date_dir = NULL; + closedir(data_dir); + data_dir = NULL; parray_qsort(backups, pgBackupCompareIdDesc); return backups; err_proc: - if (date_dir) - closedir(date_dir); + if (data_dir) + closedir(data_dir); if (backup) pgBackupFree(backup); if (backups) From 1d5f54c5eedefceacd354dea4ea5b82720ffb819 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 6 Jul 2018 19:18:05 +0300 Subject: [PATCH 55/66] PGPRO-1376: Do not call dir_is_empty() twice, because readdir() may bring different ordering --- src/restore.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/restore.c b/src/restore.c index dc339fd5..a39b5127 100644 --- a/src/restore.c +++ b/src/restore.c @@ -581,14 +581,6 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) linked_path, dir_created, link_name); } - /* - * This check was done in check_tablespace_mapping(). But do - * it again. - */ - if (!dir_is_empty(linked_path)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - if (link_sep) elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", linked_path, From 1b70807d384d71c39558959b3355cf07864077d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Jul 2018 20:40:59 +0300 Subject: [PATCH 56/66] tests: fix archive tests --- tests/archive.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 92c0fbba..97fdca64 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -46,9 +46,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) # Recreate backup calagoue @@ -66,9 +65,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=["--recovery-target-action=promote"]) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) self.assertEqual( @@ -118,9 +116,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print(node.safe_psql( @@ -153,9 +150,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: @@ -185,10 +181,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) + if self.verbose: print('Fourth timeline') print(node.safe_psql( @@ -201,8 +197,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print('Fifth timeline') @@ -216,8 +212,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: sleep(1) if self.verbose: print('Sixth timeline') @@ -463,6 +459,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica, synchronous=True) + exit(1) self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.start() @@ -772,6 +769,10 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node(backup_dir, 'node', node) node.start() + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: + sleep(1) + self.assertEqual( result, node.safe_psql( @@ -803,7 +804,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL 10 for this test') else: - pg_receivexlog_path = node.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') pg_receivexlog = self.run_binary( [ @@ -843,6 +844,11 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.cleanup() self.restore_node(backup_dir, 'node', node) node.start() + + while node.psql( + "postgres", "select pg_is_in_recovery()")[0] != 0: + sleep(1) + self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), 'data after restore not equal to original data') From e90714bcd44939c1a10eaf7d60d7a0e67852d84d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 10 Jul 2018 14:53:16 +0300 Subject: [PATCH 57/66] Push down path variable --- src/pg_probackup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e0cb5479..e6807dc0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -196,7 +196,6 @@ main(int argc, char *argv[]) { char *command = NULL, *command_name; - char path[MAXPGPATH]; /* Check if backup_path is directory. */ struct stat stat_buf; int rc; @@ -377,6 +376,8 @@ main(int argc, char *argv[]) */ if (instance_name && backup_subcmd != SET_CONFIG_CMD) { + char path[MAXPGPATH]; + /* Read environment variables */ pgut_getopt_env(options); From 963eb2e2c6bb795bf397c3ed97b1924d99287b63 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 11 Jul 2018 10:29:36 +0300 Subject: [PATCH 58/66] check for interrupt more frequently --- src/data.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/data.c b/src/data.c index fd09e5e7..476cbbdf 100644 --- a/src/data.c +++ b/src/data.c @@ -242,6 +242,10 @@ backup_data_page(backup_files_args *arguments, header.block = blknum; header.compressed_size = 0; + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + /* * Read the page and verify its header and checksum. * Under high write load it's possible that we've read partly From f98c91b3ba80fa2a149cd2fe26f3fd4d8c23be33 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 11 Jul 2018 10:50:38 +0300 Subject: [PATCH 59/66] tests: add slow_start() method to PostgresNode --- tests/archive.py | 51 ++++----------- tests/compression.py | 36 ++++++---- tests/delta.py | 10 +-- tests/expected/option_help.out | 3 + tests/false_positive.py | 4 +- tests/helpers/ptrack_helpers.py | 21 ++++++ tests/page.py | 55 ++++++---------- tests/ptrack.py | 43 +++--------- tests/restore_test.py | 112 ++++++-------------------------- tests/validate_test.py | 2 +- 10 files changed, 116 insertions(+), 221 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 97fdca64..6cd86e7d 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -45,10 +45,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() # Recreate backup calagoue self.init_pb(backup_dir) @@ -64,10 +61,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=["--recovery-target-action=promote"]) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -115,10 +109,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print(node.safe_psql( "postgres", @@ -149,10 +141,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() if self.verbose: print( @@ -180,10 +169,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() if self.verbose: print('Fourth timeline') @@ -196,10 +182,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print('Fifth timeline') print(node.safe_psql( @@ -211,10 +195,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() + if self.verbose: print('Sixth timeline') print(node.safe_psql( @@ -459,7 +441,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica, synchronous=True) - exit(1) + self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.start() @@ -768,10 +750,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, @@ -843,11 +822,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() - - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - sleep(1) + node.slow_start() self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), diff --git a/tests/compression.py b/tests/compression.py index eff42963..aa275382 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -83,7 +83,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -98,7 +99,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -113,7 +115,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -187,7 +190,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -202,7 +206,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -217,7 +222,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -294,7 +300,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -309,7 +316,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -324,7 +332,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -401,7 +410,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -416,7 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -431,7 +442,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() diff --git a/tests/delta.py b/tests/delta.py index 7cf21758..40450016 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -508,10 +508,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") @@ -946,11 +943,8 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 4080f4b8..a72aada8 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -25,6 +25,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--replica-timeout=timeout] pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-C] [--stream [-S slot-name]] [--backup-pg-log] @@ -61,10 +62,12 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup validate -B backup-dir [--instance=instance_name] [-i backup-id] [--progress] [--time=time|--xid=xid [--inclusive=boolean]] + [--recovery-target-name=target-name] [--timeline=timeline] pg_probackup show -B backup-dir [--instance=instance_name [-i backup-id]] + [--format=format] pg_probackup delete -B backup-dir --instance=instance_name [--wal] [-i backup-id | --expired] diff --git a/tests/false_positive.py b/tests/false_positive.py index 909d69fc..1884159b 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -191,7 +191,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, @@ -290,7 +290,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index af7fe766..4cdd8836 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -112,6 +112,25 @@ class ProbackupException(Exception): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +def slow_start(self): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() @@ -205,6 +224,8 @@ class ProbackupTest(object): os.makedirs(real_base_dir) node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) node.should_rm_dirs = True node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/tests/page.py b/tests/page.py index 9a7260c2..c3ca6ee9 100644 --- a/tests/page.py +++ b/tests/page.py @@ -3,7 +3,6 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess -import time module_name = 'page' @@ -33,8 +32,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir="{0}/{1}/node_restored".format(module_name, fname)) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -49,32 +47,27 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "create table t_heap tablespace somedata as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") + node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose'] - ) + options=['--log-level-file=verbose']) self.backup_node( - backup_dir, 'node', node, backup_type='page' - ) + backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -87,8 +80,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"] - ) + "--recovery-target-action=promote"]) # Physical comparison if self.paranoia: @@ -97,21 +89,17 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() - - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node_restored.slow_start() # Logical comparison result1 = node.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + result2 = node_restored.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + self.assertEqual(result1, result2) # Clean after yourself @@ -175,7 +163,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=full_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -188,7 +176,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=page_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -254,7 +242,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -271,7 +260,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -349,10 +339,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") diff --git a/tests/ptrack.py b/tests/ptrack.py index 4823acef..c2d6abff 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -586,10 +586,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -611,10 +608,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -691,11 +685,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) @@ -721,10 +711,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -1176,11 +1163,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1412,10 +1396,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", "select pg_is_in_recovery()")[0] != 0: - time.sleep(1) + restored_node.slow_start() # COMPARE LOGICAL CONTENT result_new = restored_node.safe_psql( @@ -1450,11 +1431,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from t_heap") @@ -1553,11 +1530,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", diff --git a/tests/restore_test.py b/tests/restore_test.py index fce96911..862c8662 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -53,10 +53,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery_conf = os.path.join(node.data_dir, "recovery.conf") self.assertEqual(os.path.isfile(recovery_conf), True) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -104,10 +101,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -149,11 +143,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start(params=['-t', '10']) - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '10', '-c', '2', '--no-vacuum']) @@ -181,11 +171,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -234,11 +220,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -297,11 +279,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -366,11 +344,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -420,11 +394,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -479,11 +449,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -535,11 +501,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -602,11 +564,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -674,11 +632,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): options=["-j", "4", "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -769,10 +723,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test") self.assertEqual(result[0][0], 1) @@ -802,10 +754,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.execute("postgres", "SELECT id FROM test OFFSET 1") self.assertEqual(result[0][0], 2) @@ -881,10 +830,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) @@ -935,10 +881,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -987,11 +930,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1039,10 +978,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) @@ -1097,10 +1033,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1149,10 +1082,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-name=savepoint", "--recovery-target-action=promote"]) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result_new = node.safe_psql("postgres", "select * from t_heap") res = node.psql("postgres", "select * from t_heap_1") diff --git a/tests/validate_test.py b/tests/validate_test.py index afb2305f..29800bc9 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1320,7 +1320,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) node2.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) - node2.start() + node2.slow_start() timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] From 086cd4f7d418db1dfee1a67ec477e04ac22f0417 Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 13 Jul 2018 19:30:48 +0300 Subject: [PATCH 60/66] Separated data.c => backup_data_page() into 2 functions: prepare_page() and compress_and_backup_page(). Additionally, fixed formatting in some places. (task pgpro-1726 in jira) --- src/data.c | 209 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 117 insertions(+), 92 deletions(-) diff --git a/src/data.c b/src/data.c index 476cbbdf..544fb98c 100644 --- a/src/data.c +++ b/src/data.c @@ -27,7 +27,7 @@ #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ -static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) { uLongf compressed_size = dst_size; int rc = compress2(dst, &compressed_size, src, src_size, compress_level); @@ -35,7 +35,7 @@ static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t } /* Implementation of zlib compression method */ -static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) { uLongf dest_len = dst_size; int rc = uncompress(dst, &dest_len, src, src_size); @@ -47,7 +47,7 @@ static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_ * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. */ -static size_t +static int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) { switch (alg) @@ -70,7 +70,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -static size_t +static int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) { switch (alg) @@ -101,6 +101,7 @@ typedef struct BackupPageHeader /* Special value for compressed_size field */ #define PageIsTruncated -2 +#define SkipCurrentPage -3 /* Verify page's header */ static bool @@ -134,8 +135,8 @@ static int read_page_from_file(pgFile *file, BlockNumber blknum, FILE *in, Page page, XLogRecPtr *page_lsn) { - off_t offset = blknum*BLCKSZ; - size_t read_len = 0; + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; /* read the block */ if (fseek(in, offset, SEEK_SET) != 0) @@ -216,32 +217,29 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* - * Backup the specified block from a file of a relation. - * Verify page header and checksum of the page and write it - * to the backup file. + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated */ -static void -backup_data_page(backup_files_args *arguments, +static int32 +prepare_page(backup_files_args *arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, - FILE *in, FILE *out, - pg_crc32 *crc, int *n_skipped, - BackupMode backup_mode) + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) { - BackupPageHeader header; - Page page = malloc(BLCKSZ); - Page compressed_page = NULL; - XLogRecPtr page_lsn = 0; - size_t write_buffer_size; - char write_buffer[BLCKSZ+sizeof(header)]; - - int try_again = 100; - bool page_is_valid = false; + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; - header.block = blknum; - header.compressed_size = 0; - /* check for interrupt */ if (interrupted) elog(ERROR, "Interrupted during backup"); @@ -262,7 +260,7 @@ backup_data_page(backup_files_args *arguments, if (result == 0) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; /* Page is not actually valid, but it is absent * and we're not going to reread it or validate */ page_is_valid = true; @@ -295,35 +293,38 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) { size_t page_size = 0; - - free(page); - page = NULL; - page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size); - if (page == NULL) + if (ptrack_page == NULL) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; } else if (page_size != BLCKSZ) { + free(ptrack_page); elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", file->path, absolute_blknum, BLCKSZ, page_size); } else { /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); if (is_checksum_enabled) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* get lsn from page, provided by pg_ptrack_get_block() */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && !parse_page(page, &page_lsn)) elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " "Possible risk of a memory corruption"); @@ -332,52 +333,70 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && page_lsn < prev_backup_start_lsn) { elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); (*n_skipped)++; - free(page); - return; + return SkipCurrentPage; } - if (header.compressed_size != PageIsTruncated) - { - file->read_size += BLCKSZ; + if (page_is_truncated) + return PageIsTruncated; - compressed_page = malloc(BLCKSZ); + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, compress_alg); + page, BLCKSZ, compress_alg); file->compress_alg = compress_alg; - + file->read_size += BLCKSZ; Assert (header.compressed_size <= BLCKSZ); - } - write_buffer_size = sizeof(header); - - /* - * The page was truncated. Write only header - * to know that we must truncate restored file - */ - if (header.compressed_size == PageIsTruncated) - { - memcpy(write_buffer, &header, sizeof(header)); - } - /* The page compression failed. Write it as is. */ - else if (header.compressed_size == -1) - { - header.compressed_size = BLCKSZ; - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), page, BLCKSZ); - write_buffer_size += header.compressed_size; - } - /* The page was successfully compressed */ - else if (header.compressed_size > 0) - { - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), compressed_page, header.compressed_size); - write_buffer_size += MAXALIGN(header.compressed_size); + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } } /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", @@ -389,7 +408,7 @@ backup_data_page(backup_files_args *arguments, /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", @@ -397,11 +416,6 @@ backup_data_page(backup_files_args *arguments, } file->write_size += write_buffer_size; - - if (page != NULL) - free(page); - if (compressed_page != NULL) - free(compressed_page); } /* @@ -418,13 +432,15 @@ backup_data_file(backup_files_args* arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode) { - char to_path[MAXPGPATH]; - FILE *in; - FILE *out; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int n_blocks_skipped = 0; - int n_blocks_read = 0; + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; /* * Skip unchanged file only if it exists in previous backup. @@ -503,9 +519,11 @@ backup_data_file(backup_files_args* arguments, { for (blknum = 0; blknum < nblocks; blknum++) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page); n_blocks_read++; } if (backup_mode == BACKUP_MODE_DIFF_DELTA) @@ -518,9 +536,11 @@ backup_data_file(backup_files_args* arguments, iter = datapagemap_iterate(&file->pagemap); while (datapagemap_next(iter, &blknum)) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page); n_blocks_read++; } @@ -635,7 +655,8 @@ restore_data_file(const char *from_root, } if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u",file->path, blknum); + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); if (header.compressed_size == PageIsTruncated) { @@ -646,7 +667,8 @@ restore_data_file(const char *from_root, if (ftruncate(fileno(out), header.block * BLCKSZ) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); - elog(VERBOSE, "truncate file %s to block %u", file->path, header.block); + elog(VERBOSE, "truncate file %s to block %u", + file->path, header.block); break; } @@ -664,10 +686,12 @@ restore_data_file(const char *from_root, uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, - header.compressed_size, file->compress_alg); + header.compressed_size, + file->compress_alg); if (uncompressed_size != BLCKSZ) - elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", uncompressed_size); + elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", + uncompressed_size); } /* @@ -714,7 +738,8 @@ restore_data_file(const char *from_root, if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); - elog(INFO, "Delta truncate file %s to block %u", file->path, file->n_blocks); + elog(INFO, "Delta truncate file %s to block %u", + file->path, file->n_blocks); } } From 2a1181ee92e42665a830277d3272e2451bf33c35 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Jul 2018 01:17:04 +0300 Subject: [PATCH 61/66] tests: minor fix --- tests/ptrack_cluster.py | 8 ++++---- tests/ptrack_truncate.py | 4 ++-- tests/ptrack_vacuum.py | 2 +- tests/ptrack_vacuum_bits_frozen.py | 4 ++-- tests/ptrack_vacuum_full.py | 2 +- tests/ptrack_vacuum_truncate.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index cf8702b8..784751ef 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -46,7 +46,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_btree') @@ -103,7 +103,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_gist') @@ -172,7 +172,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') @@ -242,7 +242,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py index 51e90503..928608c4 100644 --- a/tests/ptrack_truncate.py +++ b/tests/ptrack_truncate.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -100,7 +100,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index ad2f65bc..0409cae3 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index dc1e5fd8..f0cd3bbd 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -43,7 +43,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') @@ -111,7 +111,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'vacuum freeze t_heap') diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index a49d6075..ec12c9e2 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'vacuum full t_heap') diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 5fd85056..5c84c7e8 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id > 128;') node.safe_psql('postgres', 'vacuum t_heap') @@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id > 128;') From b67c0b54c4ec435b67136142d975412879b7290f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Jul 2018 02:45:17 +0300 Subject: [PATCH 62/66] add replica flag in slow_start() method --- tests/archive.py | 26 +++++++++++++------------- tests/helpers/ptrack_helpers.py | 28 +++++++++++++++++++--------- tests/replica.py | 26 +++++++++----------------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 6cd86e7d..8b8eb71a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -29,7 +29,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -89,7 +89,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FIRST TIMELINE node.safe_psql( @@ -247,7 +247,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.append_conf( 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( archive_script_path)) - node.start() + node.slow_start() try: self.backup_node( backup_dir, 'node', node, @@ -308,7 +308,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -368,7 +368,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -423,7 +423,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -444,7 +444,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -481,7 +481,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -513,7 +513,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -547,7 +547,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -573,7 +573,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -628,7 +628,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -655,7 +655,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA # self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 4cdd8836..78886455 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -112,7 +112,7 @@ class ProbackupException(Exception): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) -def slow_start(self): +def slow_start(self, replica=False): # wait for https://github.com/postgrespro/testgres/pull/50 # self.poll_query_until( @@ -121,14 +121,24 @@ def slow_start(self): # raise_operational_error=False) self.start() - while True: - try: - self.poll_query_until( - "postgres", - "SELECT not pg_is_in_recovery()") - break - except Exception as e: - continue + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + break + except Exception as e: + continue class ProbackupTest(object): diff --git a/tests/replica.py b/tests/replica.py index 5d6ecfb1..d95d0139 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -50,7 +50,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_replica(master, replica) # Check data correctness on replica - replica.start(["-t", "600"]) + replica.slow_start(replica=True) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -82,7 +82,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -113,7 +113,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -143,7 +143,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -166,7 +166,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start(["-t", "600"]) + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -200,7 +200,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -231,7 +231,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -260,7 +260,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -287,15 +287,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.start(["-t", "600"]) - - time.sleep(1) - self.assertEqual( - master.safe_psql( - "postgres", - "select exists(select * from pg_stat_replication)" - ).rstrip(), - 't') + replica.slow_start(replica=True) # Clean after yourself self.del_test_dir(module_name, fname) From f0864a2d4388ab75956b5965611db3decd4c6341 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Jul 2018 16:37:48 +0300 Subject: [PATCH 63/66] tests: replica slow_start() minor change --- tests/__init__.py | 4 ++++ tests/helpers/ptrack_helpers.py | 20 ++++++++++++-------- tests/replica.py | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d5f8b248..aeeabf2a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -60,6 +60,10 @@ def load_tests(loader, tests, pattern): # ptrack backup on replica should work correctly # archive: # immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored # 10vanilla_1.3ptrack + # 10vanilla+ # 9.6vanilla_1.3ptrack + diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 78886455..c430738c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -131,14 +131,18 @@ def slow_start(self, replica=False): except Exception as e: continue else: - while True: - try: - self.poll_query_until( - "postgres", - "SELECT pg_is_in_recovery()") - break - except Exception as e: - continue + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue class ProbackupTest(object): diff --git a/tests/replica.py b/tests/replica.py index d95d0139..d74c375c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -287,7 +287,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.slow_start(replica=True) + replica.start() # Clean after yourself self.del_test_dir(module_name, fname) From 4877f6b5d48842e6f9ead7f42431225d451fcdef Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 14:22:54 +0300 Subject: [PATCH 64/66] Issue #31: Use UTC time to make backup ID --- src/backup.c | 2 +- src/catalog.c | 6 +++--- src/delete.c | 2 +- src/parsexlog.c | 9 +++++---- src/pg_probackup.h | 2 +- src/restore.c | 5 +++-- src/show.c | 10 ++++++---- src/util.c | 26 +++++++++++++++----------- 8 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/backup.c b/src/backup.c index 420bcb78..666e8acc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -553,7 +553,7 @@ do_backup_instance(void) pg_ptrack_clear(); /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time); + time2iso(label, lengthof(label), current.start_time, true); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); diff --git a/src/catalog.c b/src/catalog.c index a0a8e5dc..9b3339dd 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -409,17 +409,17 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); fprintf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, false); fprintf(out, "end-time = '%s'\n", timestamp); } fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); fprintf(out, "recovery-time = '%s'\n", timestamp); } diff --git a/src/delete.c b/src/delete.c index f81fe70d..5b5c79c7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -257,7 +257,7 @@ pgBackupDeleteFiles(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) return 0; - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, true); elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); diff --git a/src/parsexlog.c b/src/parsexlog.c index a0079b42..a83acfe6 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -380,7 +380,8 @@ validate_wal(pgBackup *backup, xlogfpath[0] = '\0'; /* We can restore at least up to the backup end */ - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, + true); last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -430,7 +431,7 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time)); + timestamptz_to_time_t(last_time), true); /* There are all needed WAL records */ if (all_wal) @@ -459,8 +460,8 @@ validate_wal(pgBackup *backup, last_timestamp, last_xid); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time, + true); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 75788cd7..0a3be70b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -521,7 +521,7 @@ extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern void sanityChecks(void); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool to_local); extern const char *status2str(BackupStatus status); extern void remove_trailing_space(char *buf, int comment_mark); extern void remove_not_digit(char *buf, size_t len, const char *str); diff --git a/src/restore.c b/src/restore.c index a39b5127..1c295d23 100644 --- a/src/restore.c +++ b/src/restore.c @@ -122,7 +122,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup = (pgBackup *) parray_get(backups, i); /* Skip all backups which started after target backup */ - if (target_backup_id && current_backup->start_time > target_backup_id) + if (target_backup_id != INVALID_BACKUP_ID && + current_backup->start_time > target_backup_id) continue; /* @@ -381,7 +382,7 @@ restore_backup(pgBackup *backup) "XLOG_BLCKSZ(%d) is not compatible(%d expected)", backup->wal_block_size, XLOG_BLCKSZ); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, true); elog(LOG, "restoring database from backup %s", timestamp); /* diff --git a/src/show.c b/src/show.c index 01a558e0..d783ba06 100644 --- a/src/show.c +++ b/src/show.c @@ -324,7 +324,8 @@ show_instance_plain(parray *backup_list, bool show_name) char data_bytes_str[10] = "----"; if (backup->recovery_time != (time_t) 0) - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, + true); if (backup->end_time != (time_t) 0) snprintf(duration, lengthof(duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); @@ -452,12 +453,12 @@ show_instance_json(parray *backup_list) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, true); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, true); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -466,7 +467,8 @@ show_instance_json(parray *backup_list) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, + true); json_add_value(buf, "recovery-time", timestamp, json_level, true); } diff --git a/src/util.c b/src/util.c index a43239dc..7893f3da 100644 --- a/src/util.c +++ b/src/util.c @@ -194,31 +194,35 @@ get_data_checksum_version(bool safe) * Convert time_t value to ISO-8601 format string */ void -time2iso(char *buf, size_t len, time_t time) +time2iso(char *buf, size_t len, time_t time, bool to_local) { struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; - ptm = localtime(&time); - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); - - if (offset != 0) + if (to_local) { - buf += strlen(buf); - sprintf(buf, "%c%02d", + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", (offset >= 0) ? '+' : '-', abs((int) offset) / SECS_PER_HOUR); if (abs((int) offset) % SECS_PER_HOUR != 0) { - buf += strlen(buf); - sprintf(buf, ":%02d", + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } + else + strftime(buf, len, "%Y-%m-%d %H:%M:%S+00", ptm); } /* copied from timestamp.c */ From 188bb1119642872aea0606ee86ad3f7fdfb5899b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 15:02:50 +0300 Subject: [PATCH 65/66] Revert "Issue #31: Use UTC time to make backup ID" This reverts commit 4877f6b5d48842e6f9ead7f42431225d451fcdef. --- src/backup.c | 2 +- src/catalog.c | 6 +++--- src/delete.c | 2 +- src/parsexlog.c | 9 ++++----- src/pg_probackup.h | 2 +- src/restore.c | 5 ++--- src/show.c | 10 ++++------ src/util.c | 26 +++++++++++--------------- 8 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 666e8acc..420bcb78 100644 --- a/src/backup.c +++ b/src/backup.c @@ -553,7 +553,7 @@ do_backup_instance(void) pg_ptrack_clear(); /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time, true); + time2iso(label, lengthof(label), current.start_time); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); diff --git a/src/catalog.c b/src/catalog.c index 9b3339dd..a0a8e5dc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -409,17 +409,17 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time, false); + time2iso(timestamp, lengthof(timestamp), backup->start_time); fprintf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time, false); + time2iso(timestamp, lengthof(timestamp), backup->end_time); fprintf(out, "end-time = '%s'\n", timestamp); } fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); fprintf(out, "recovery-time = '%s'\n", timestamp); } diff --git a/src/delete.c b/src/delete.c index 5b5c79c7..f81fe70d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -257,7 +257,7 @@ pgBackupDeleteFiles(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) return 0; - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); diff --git a/src/parsexlog.c b/src/parsexlog.c index a83acfe6..a0079b42 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -380,8 +380,7 @@ validate_wal(pgBackup *backup, xlogfpath[0] = '\0'; /* We can restore at least up to the backup end */ - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, - true); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -431,7 +430,7 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time), true); + timestamptz_to_time_t(last_time)); /* There are all needed WAL records */ if (all_wal) @@ -460,8 +459,8 @@ validate_wal(pgBackup *backup, last_timestamp, last_xid); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), target_time, - true); + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0a3be70b..75788cd7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -521,7 +521,7 @@ extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern void sanityChecks(void); -extern void time2iso(char *buf, size_t len, time_t time, bool to_local); +extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); extern void remove_trailing_space(char *buf, int comment_mark); extern void remove_not_digit(char *buf, size_t len, const char *str); diff --git a/src/restore.c b/src/restore.c index 1c295d23..a39b5127 100644 --- a/src/restore.c +++ b/src/restore.c @@ -122,8 +122,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup = (pgBackup *) parray_get(backups, i); /* Skip all backups which started after target backup */ - if (target_backup_id != INVALID_BACKUP_ID && - current_backup->start_time > target_backup_id) + if (target_backup_id && current_backup->start_time > target_backup_id) continue; /* @@ -382,7 +381,7 @@ restore_backup(pgBackup *backup) "XLOG_BLCKSZ(%d) is not compatible(%d expected)", backup->wal_block_size, XLOG_BLCKSZ); - time2iso(timestamp, lengthof(timestamp), backup->start_time, true); + time2iso(timestamp, lengthof(timestamp), backup->start_time); elog(LOG, "restoring database from backup %s", timestamp); /* diff --git a/src/show.c b/src/show.c index d783ba06..01a558e0 100644 --- a/src/show.c +++ b/src/show.c @@ -324,8 +324,7 @@ show_instance_plain(parray *backup_list, bool show_name) char data_bytes_str[10] = "----"; if (backup->recovery_time != (time_t) 0) - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, - true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); if (backup->end_time != (time_t) 0) snprintf(duration, lengthof(duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); @@ -453,12 +452,12 @@ show_instance_json(parray *backup_list) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time, true); + time2iso(timestamp, lengthof(timestamp), backup->start_time); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time, true); + time2iso(timestamp, lengthof(timestamp), backup->end_time); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -467,8 +466,7 @@ show_instance_json(parray *backup_list) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, - true); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); json_add_value(buf, "recovery-time", timestamp, json_level, true); } diff --git a/src/util.c b/src/util.c index 7893f3da..a43239dc 100644 --- a/src/util.c +++ b/src/util.c @@ -194,35 +194,31 @@ get_data_checksum_version(bool safe) * Convert time_t value to ISO-8601 format string */ void -time2iso(char *buf, size_t len, time_t time, bool to_local) +time2iso(char *buf, size_t len, time_t time) { struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; - if (to_local) + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); + + if (offset != 0) { - char *ptr = buf; - - ptm = localtime(&time); - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), "%c%02d", + buf += strlen(buf); + sprintf(buf, "%c%02d", (offset >= 0) ? '+' : '-', abs((int) offset) / SECS_PER_HOUR); if (abs((int) offset) % SECS_PER_HOUR != 0) { - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), ":%02d", + buf += strlen(buf); + sprintf(buf, ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } - else - strftime(buf, len, "%Y-%m-%d %H:%M:%S+00", ptm); } /* copied from timestamp.c */ From 059d3845cb080a7d3407d8591b546b9750578026 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 16 Jul 2018 15:18:08 +0300 Subject: [PATCH 66/66] Issue #31: Times in backup.control is in UTC by default --- src/backup.c | 2 +- src/restore.c | 2 +- src/util.c | 26 ++++++++++++-------------- src/utils/pgut.c | 10 +++++++--- src/utils/pgut.h | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index 420bcb78..4b9bb1ff 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1836,7 +1836,7 @@ pg_stop_backup(pgBackup *backup) elog(ERROR, "result of txid_snapshot_xmax() is invalid: %s", PQgetvalue(res, 0, 0)); - if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time)) + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) elog(ERROR, "result of current_timestamp is invalid: %s", PQgetvalue(res, 0, 1)); diff --git a/src/restore.c b/src/restore.c index a39b5127..80bbb8bc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1036,7 +1036,7 @@ parseRecoveryTargetOptions(const char *target_time, rt->time_specified = true; rt->target_time_string = target_time; - if (parse_time(target_time, &dummy_time)) + if (parse_time(target_time, &dummy_time, false)) rt->recovery_target_time = dummy_time; else elog(ERROR, "Invalid value of --time option %s", target_time); diff --git a/src/util.c b/src/util.c index a43239dc..051f4d54 100644 --- a/src/util.c +++ b/src/util.c @@ -191,7 +191,7 @@ get_data_checksum_version(bool safe) /* - * Convert time_t value to ISO-8601 format string + * Convert time_t value to ISO-8601 format string. Always set timezone offset. */ void time2iso(char *buf, size_t len, time_t time) @@ -199,25 +199,23 @@ time2iso(char *buf, size_t len, time_t time) struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; + char *ptr = buf; ptm = localtime(&time); offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - if (offset != 0) + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) { - buf += strlen(buf); - sprintf(buf, "%c%02d", - (offset >= 0) ? '+' : '-', - abs((int) offset) / SECS_PER_HOUR); - - if (abs((int) offset) % SECS_PER_HOUR != 0) - { - buf += strlen(buf); - sprintf(buf, ":%02d", - abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); - } + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index f1b5babd..330f2fa5 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -256,7 +256,8 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) message = "a valid string. But provided: "; break; case 't': - if (parse_time(optarg, opt->var)) + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) return; message = "a time"; break; @@ -746,9 +747,12 @@ parse_uint64(const char *value, uint64 *result, int flags) /* * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. */ bool -parse_time(const char *value, time_t *result) +parse_time(const char *value, time_t *result, bool utc_default) { size_t len; int fields_num, @@ -870,7 +874,7 @@ parse_time(const char *value, time_t *result) *result = mktime(&tm); /* adjust time zone */ - if (tz_set) + if (tz_set || utc_default) { time_t ltime = time(NULL); struct tm *ptm = gmtime(<ime); diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a27cea83..662f07c1 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -204,7 +204,7 @@ extern bool parse_int32(const char *value, int32 *result, int flags); extern bool parse_uint32(const char *value, uint32 *result, int flags); extern bool parse_int64(const char *value, int64 *result, int flags); extern bool parse_uint64(const char *value, uint64 *result, int flags); -extern bool parse_time(const char *value, time_t *result); +extern bool parse_time(const char *value, time_t *result, bool utc_default); extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg);