diff --git a/Makefile b/Makefile index 0f4fa672..e22bc86c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ src/pg_probackup.o src/restore.o src/show.o src/status.o \ src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ src/xlogreader.o src/streamutil.o src/receivelog.o \ - src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o src/utils/thread.o src/merge.o EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h @@ -62,6 +63,7 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + ifeq ($(MAJORVERSION),10) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 00000000..b46e3b36 --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 00000000..94d242c9 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 00000000..3ea79e96 --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 00000000..2df4b404 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 00000000..46a7b2c2 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,212 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 00000000..46e019ba --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 00000000..2fc101a4 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index a0b02fcf..9a4c7005 100644 --- a/src/backup.c +++ b/src/backup.c @@ -18,16 +18,16 @@ #include #include #include -#include -#include "libpq/pqsignal.h" -#include "storage/bufpage.h" #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" #include "datapagemap.h" -#include "receivelog.h" -#include "streamutil.h" +#include "libpq/pqsignal.h" #include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; @@ -46,6 +46,9 @@ const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + /* * We need to wait end of WAL streaming before execute pg_stop_backup(). */ @@ -89,8 +92,8 @@ static bool pg_stop_backup_is_sent = false; static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -static void backup_files(void *arg); -static void remote_backup_files(void *arg); +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); static void do_backup_instance(void); @@ -101,11 +104,10 @@ static int checkpoint_timeout(void); //static void backup_list_file(parray *files, const char *root, ) static void parse_backup_filelist_filenames(parray *files, const char *root); -static void write_backup_file_list(parray *files, const char *root); static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); -static void StreamLog(void *arg); +static void *StreamLog(void *arg); static void get_remote_pgdata_filelist(parray *files); static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); @@ -130,7 +132,6 @@ static void check_system_identifiers(void); static void confirm_block_size(const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); - #define disconnect_and_exit(code) \ { \ if (conn != NULL) PQfinish(conn); \ @@ -253,7 +254,11 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) else if (copybuf[156] == '2') { /* Symlink */ +#ifndef WIN32 pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif } else elog(ERROR, "Unrecognized link indicator \"%c\"\n", @@ -289,7 +294,7 @@ remote_copy_file(PGconn *conn, pgFile* file) DATABASE_DIR); join_path_components(to_path, database_path, file->path); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -323,7 +328,7 @@ remote_copy_file(PGconn *conn, pgFile* file) { write_buffer_size = Min(row_length, sizeof(buf)); memcpy(buf, copybuf, write_buffer_size); - COMP_CRC32C(file->crc, &buf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); /* TODO calc checksum*/ if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) @@ -353,7 +358,7 @@ remote_copy_file(PGconn *conn, pgFile* file) elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); } - file->write_size = file->read_size; + file->write_size = (int64) file->read_size; FIN_CRC32C(file->crc); fclose(out); @@ -363,13 +368,13 @@ remote_copy_file(PGconn *conn, pgFile* file) * Take a remote backup of the PGDATA at a file level. * Copy all directories and files listed in backup_files_list. */ -static void +static void * remote_backup_files(void *arg) { - int i; - backup_files_args *arguments = (backup_files_args *) arg; - int n_backup_files_list = parray_num(arguments->backup_files_list); - PGconn *file_backup_conn = NULL; + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; for (i = 0; i < n_backup_files_list; i++) { @@ -379,13 +384,13 @@ remote_backup_files(void *arg) pgFile *file; int row_length; - file = (pgFile *) parray_get(arguments->backup_files_list, i); + file = (pgFile *) parray_get(arguments->files_list, i); /* We have already copied all directories */ if (S_ISDIR(file->mode)) continue; - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; file_backup_conn = pgut_connect_replication(pgut_dbname); @@ -434,13 +439,15 @@ remote_backup_files(void *arg) /* receive the data from stream and write to backup file */ remote_copy_file(file_backup_conn, file); - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", + file->path, file->write_size); PQfinish(file_backup_conn); } /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* @@ -456,12 +463,12 @@ do_backup_instance(void) char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - pthread_t backup_threads[num_threads]; - backup_files_args *backup_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; bool backup_isok = true; pgBackup *prev_backup = NULL; - char prev_backup_filelist_path[MAXPGPATH]; parray *prev_backup_filelist = NULL; elog(LOG, "Database backup start"); @@ -493,6 +500,7 @@ do_backup_instance(void) } else current.tli = get_current_timeline(false); + /* * In incremental backup mode ensure that already-validated * backup on current timeline exists and get its filelist. @@ -502,10 +510,10 @@ do_backup_instance(void) current.backup_mode == BACKUP_MODE_DIFF_DELTA) { parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; + /* get list of backups already taken */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); prev_backup = catalog_get_last_data_backup(backup_list, current.tli); if (prev_backup == NULL) @@ -513,8 +521,8 @@ do_backup_instance(void) "Create new FULL backup before an incremental one."); parray_free(backup_list); - pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), - DATABASE_FILE_LIST); + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); @@ -592,8 +600,7 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; - pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, - &stream_thread_arg); + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } /* initialize backup list */ @@ -605,6 +612,19 @@ do_backup_instance(void) else dir_list_file(backup_files_list, pgdata, true, true, false); + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); @@ -633,28 +653,18 @@ do_backup_instance(void) * For backup from master wait for previous segment. * For backup from replica wait for current segment. */ - !from_replica, backup_files_list); + !current.from_replica, backup_files_list); } - - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - parray_qsort(backup_files_list, pgFileComparePath); + /* + * Build the page map from ptrack information. + */ make_pagemap_from_ptrack(backup_files_list); } /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - */ - parray_qsort(backup_files_list, pgFileComparePath); - - /* - * Make directories before backup - * and setup threads at the same time + * Make directories before backup and setup threads at the same time */ for (i = 0; i < parray_num(backup_files_list); i++) { @@ -681,53 +691,54 @@ do_backup_instance(void) } /* setup threads */ - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } - /* sort by size for load balancing */ + /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + for (i = 0; i < num_threads; i++) { - backup_files_args *arg = pg_malloc(sizeof(backup_files_args)); + backup_files_arg *arg = &(threads_args[i]); arg->from_root = pgdata; arg->to_root = database_path; - arg->backup_files_list = backup_files_list; - arg->prev_backup_filelist = prev_backup_filelist; - arg->prev_backup_start_lsn = prev_backup_start_lsn; - arg->thread_backup_conn = NULL; - arg->thread_cancel_conn = NULL; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; /* By default there are some error */ arg->ret = 1; - backup_threads_args[i] = arg; } /* Run threads */ elog(LOG, "Start transfering data files"); for (i = 0; i < num_threads; i++) { + backup_files_arg *arg = &(threads_args[i]); + elog(VERBOSE, "Start thread num: %i", i); if (!is_remote_backup) - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) backup_files, - backup_threads_args[i]); + pthread_create(&threads[i], NULL, backup_files, arg); else - pthread_create(&backup_threads[i], NULL, - (void *(*)(void *)) remote_backup_files, - backup_threads_args[i]); + pthread_create(&threads[i], NULL, remote_backup_files, arg); } /* Wait threads */ for (i = 0; i < num_threads; i++) { - pthread_join(backup_threads[i], NULL); - if (backup_threads_args[i]->ret == 1) + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) backup_isok = false; - - pg_free(backup_threads_args[i]); } if (backup_isok) elog(LOG, "Data files are transfered"); @@ -758,12 +769,14 @@ do_backup_instance(void) for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + if (S_ISREG(file->mode)) calc_file_checksum(file); /* Remove file path root prefix*/ if (strstr(file->path, database_path) == file->path) { char *ptr = file->path; + file->path = pstrdup(GetRelativePath(ptr, database_path)); free(ptr); } @@ -775,7 +788,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - write_backup_file_list(backup_files_list, pgdata); + pgBackupWriteFileList(¤t, backup_files_list, pgdata); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -815,11 +828,15 @@ do_backup(time_t start_time) pgut_atexit_push(backup_disconnect, NULL); current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); confirm_block_size("wal_block_size", XLOG_BLCKSZ); - from_replica = pg_is_in_recovery(); + current.from_replica = pg_is_in_recovery(); /* Confirm that this server version is supported */ check_server_version(); @@ -837,7 +854,7 @@ do_backup(time_t start_time) elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " "pg_probackup have no way to detect data block corruption without them. " "Reinitialize PGDATA with option '--data-checksums'."); - + StrNCpy(current.server_version, server_version_str, sizeof(current.server_version)); current.stream = stream_wal; @@ -859,7 +876,7 @@ do_backup(time_t start_time) } } - if (from_replica) + if (current.from_replica) { /* Check master connection options */ if (master_host == NULL) @@ -956,7 +973,7 @@ check_server_version(void) "server version is %s, must be %s or higher", server_version_str, "9.5"); - if (from_replica && server_version < 90600) + if (current.from_replica && server_version < 90600) elog(ERROR, "server version is %s, must be %s or higher for backup from replica", server_version_str, "9.6"); @@ -972,23 +989,23 @@ check_server_version(void) if (PQresultStatus(res) == PGRES_FATAL_ERROR) /* It seems we connected to PostgreSQL (not Postgres Pro) */ elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection made with PostgreSQL %s", + "but connection is made with PostgreSQL %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection made with Postgres Pro %s %s", + "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str, PQgetvalue(res, 0, 0)); #else if (PQresultStatus(res) != PGRES_FATAL_ERROR) /* It seems we connected to Postgres Pro (not PostgreSQL) */ elog(ERROR, "%s was built with PostgreSQL %s, " - "but connection made with Postgres Pro %s %s", + "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, server_version_str, PQgetvalue(res, 0, 0)); else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) - elog(ERROR, "%s was built with PostgreSQL %s, but connection made with %s", + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", PROGRAM_NAME, PG_MAJORVERSION, server_version_str); #endif @@ -1038,12 +1055,12 @@ confirm_block_size(const char *name, int blcksz) elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - PQclear(res); - if ((endp && *endp) || block_size != blcksz) elog(ERROR, "%s(%d) is not compatible(%d expected)", name, block_size, blcksz); + + PQclear(res); } /* @@ -1061,7 +1078,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; /* For replica we call pg_start_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (backup->from_replica) ? master_conn : backup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1076,6 +1093,12 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) 2, params); + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ @@ -1106,14 +1129,8 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) } /* Wait for start_lsn to be replayed by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(backup->start_lsn, true); - - /* - * Set flag that pg_start_backup() was called. If an error will happen it - * is necessary to call pg_stop_backup() in backup_cleanup(). - */ - backup_in_progress = true; } /* @@ -1555,8 +1572,6 @@ wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) { uint32 try_count = 0; - Assert(from_replica); - while (true) { PGresult *res; @@ -1651,7 +1666,7 @@ pg_stop_backup(pgBackup *backup) elog(FATAL, "backup is not in progress"); /* For replica we call pg_stop_backup() on master */ - conn = (from_replica) ? master_conn : backup_conn; + conn = (current.from_replica) ? master_conn : backup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", @@ -1664,7 +1679,7 @@ pg_stop_backup(pgBackup *backup) const char *params[1]; char name[1024]; - if (!from_replica) + if (!current.from_replica) snprintf(name, lengthof(name), "pg_probackup, backup_id %s", base36enc(backup->start_time)); else @@ -1800,7 +1815,7 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, "w"); + fp = fopen(backup_label, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1831,7 +1846,7 @@ pg_stop_backup(pgBackup *backup) elog(ERROR, "result of txid_snapshot_xmax() is invalid: %s", PQgetvalue(res, 0, 0)); - if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time)) + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) elog(ERROR, "result of current_timestamp is invalid: %s", PQgetvalue(res, 0, 1)); @@ -1848,7 +1863,7 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, "w"); + fp = fopen(tablespace_map, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); @@ -1892,7 +1907,7 @@ pg_stop_backup(pgBackup *backup) stream_xlog_path[MAXPGPATH]; /* Wait for stop_lsn to be received by replica */ - if (from_replica) + if (backup->from_replica) wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. @@ -2004,22 +2019,22 @@ backup_disconnect(bool fatal, void *userdata) * In incremental backup mode, copy only files or datafiles' pages changed after * previous backup. */ -static void +static void * backup_files(void *arg) { - int i; - backup_files_args *arguments = (backup_files_args *) arg; - int n_backup_files_list = parray_num(arguments->backup_files_list); + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); /* backup a file */ for (i = 0; i < n_backup_files_list; i++) { int ret; struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i); elog(VERBOSE, "Copying file: \"%s\" ", file->path); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ @@ -2061,69 +2076,69 @@ backup_files(void *arg) /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) { - int p; char *relative; - int n_prev_backup_files_list = parray_num(arguments->prev_backup_filelist); + pgFile key; + pgFile **prev_file; + relative = GetRelativePath(file->path, arguments->from_root); - for (p = 0; p < n_prev_backup_files_list; p++) - { - pgFile *prev_file = (pgFile *) parray_get(arguments->prev_backup_filelist, p); - if (strcmp(relative, prev_file->path) == 0) - { - /* File exists in previous backup */ - file->exists_in_prev = true; - // elog(VERBOSE, "File exists at the time of previous backup %s", relative); - break; - } - } + key.path = relative; + + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; } /* copy the file into backup */ if (file->is_datafile && !file->is_cfs) { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + /* backup block by block if datafile AND not compressed by cfs*/ - if (!backup_data_file(arguments, - arguments->from_root, - arguments->to_root, file, - arguments->prev_backup_start_lsn, - current.backup_mode)) + if (!backup_data_file(arguments, to_path, file, + arguments->prev_start_lsn, + current.backup_mode, + compress_alg, compress_level)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; } } - else - /* TODO: - * Check if file exists in previous backup - * If exists: - * if mtime > start_backup_time of parent backup, - * copy file to backup - * if mtime < start_backup_time - * calculate crc, compare crc to old file - * if crc is the same -> skip file - */ - if (!copy_file(arguments->from_root, - arguments->to_root, - file)) + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; } - elog(VERBOSE, "File \"%s\". Copied %lu bytes", - file->path, (unsigned long) file->write_size); + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + file->path, file->write_size); } else elog(LOG, "unexpected file type %d", buf.st_mode); } /* Close connection */ - if (arguments->thread_backup_conn) - pgut_disconnect(arguments->thread_backup_conn); + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); /* Data files transferring is successful */ arguments->ret = 0; + + return NULL; } /* @@ -2261,52 +2276,6 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) free(cfs_tblspc_path); } - -/* - * Output the list of files to backup catalog DATABASE_FILE_LIST - */ -static void -write_backup_file_list(parray *files, const char *root) -{ - FILE *fp; - char path[MAXPGPATH]; - - pgBackupGetPath(¤t, path, lengthof(path), DATABASE_FILE_LIST); - - fp = fopen(path, "wt"); - if (fp == NULL) - elog(ERROR, "cannot open file list \"%s\": %s", path, - strerror(errno)); - - print_file_list(fp, files, root); - - if (fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) - elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); -} - -/* - * A helper function to create the path of a relation file and segment. - * The returned path is palloc'd - */ -static char * -datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno) -{ - char *path; - char *segpath; - - path = relpathperm(rnode, forknum); - if (segno > 0) - { - segpath = psprintf("%s.%u", path, segno); - pfree(path); - return segpath; - } - else - return path; -} - /* * Find pgfile by given rnode in the backup_files_list * and add given blkno to its pagemap. @@ -2314,30 +2283,28 @@ datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno) void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) { - char *path; - char *rel_path; + char *path; + char *rel_path; BlockNumber blkno_inseg; int segno; - pgFile *file_item = NULL; - int j; + pgFile **file_item; + pgFile f; segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; - rel_path = datasegpath(rnode, forknum, segno); - path = pg_malloc(strlen(rel_path) + strlen(pgdata) + 2); - sprintf(path, "%s/%s", pgdata, rel_path); + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + else + path = psprintf("%s/%s", pgdata, rel_path); - for (j = 0; j < parray_num(backup_files_list); j++) - { - pgFile *p = (pgFile *) parray_get(backup_files_list, j); + pg_free(rel_path); - if (strcmp(p->path, path) == 0) - { - file_item = p; - break; - } - } + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); /* * If we don't have any record of this file in the file map, it means @@ -2346,10 +2313,18 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) * backup would simply copy it as-is. */ if (file_item) - datapagemap_add(&file_item->pagemap, blkno_inseg); + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_lock(&backup_pagemap_mutex); + + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } pg_free(path); - pg_free(rel_path); } /* @@ -2399,12 +2374,12 @@ make_pagemap_from_ptrack(parray *files) if (file->is_datafile) { - if (file->tblspcOid == tblspcOid_with_ptrack_init - && file->dbOid == dbOid_with_ptrack_init) + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) { /* ignore ptrack if ptrack_init exists */ elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; continue; } @@ -2432,19 +2407,32 @@ make_pagemap_from_ptrack(parray *files) */ start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; - if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) { - file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; } else { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); - memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } } else { @@ -2457,7 +2445,7 @@ make_pagemap_from_ptrack(parray *files) * - target relation was deleted. */ elog(VERBOSE, "Ptrack is missing for file: %s", file->path); - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap_isabsent = true; } } } @@ -2533,7 +2521,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* * Start the log streaming */ -static void +static void * StreamLog(void *arg) { XLogRecPtr startpos; @@ -2607,6 +2595,8 @@ StreamLog(void *arg) PQfinish(stream_arg->conn); stream_arg->conn = NULL; + + return NULL; } /* @@ -2633,7 +2623,7 @@ get_last_ptrack_lsn(void) } char * -pg_ptrack_get_block(backup_files_args *arguments, +pg_ptrack_get_block(backup_files_arg *arguments, Oid dbOid, Oid tblsOid, Oid relOid, @@ -2658,17 +2648,17 @@ pg_ptrack_get_block(backup_files_args *arguments, sprintf(params[2], "%i", relOid); sprintf(params[3], "%u", blknum); - if (arguments->thread_backup_conn == NULL) + if (arguments->backup_conn == NULL) { - arguments->thread_backup_conn = pgut_connect(pgut_dbname); + arguments->backup_conn = pgut_connect(pgut_dbname); } - if (arguments->thread_cancel_conn == NULL) - arguments->thread_cancel_conn = PQgetCancel(arguments->thread_backup_conn); + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - res = pgut_execute_parallel(arguments->thread_backup_conn, - arguments->thread_cancel_conn, + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", 4, (const char **)params, true); diff --git a/src/catalog.c b/src/catalog.c index f5884f01..51d791a7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -251,14 +250,15 @@ IsDir(const char *dirpath, const char *entry) parray * catalog_get_backup_list(time_t requested_backup_id) { - DIR *date_dir = NULL; - struct dirent *date_ent = NULL; + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; parray *backups = NULL; pgBackup *backup = NULL; + int i; /* open backup instance backups directory */ - date_dir = opendir(backup_instance_path); - if (date_dir == NULL) + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) { elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, strerror(errno)); @@ -267,22 +267,23 @@ catalog_get_backup_list(time_t requested_backup_id) /* scan the directory and list backups */ backups = parray_new(); - for (; (date_ent = readdir(date_dir)) != NULL; errno = 0) + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) { char backup_conf_path[MAXPGPATH]; - char date_path[MAXPGPATH]; + char data_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, date_ent->d_name) - || date_ent->d_name[0] == '.') + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(date_path, backup_instance_path, date_ent->d_name); + join_path_components(data_path, backup_instance_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ - snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); + backup->backup_id = backup->start_time; /* ignore corrupted backups */ if (backup) @@ -299,8 +300,8 @@ catalog_get_backup_list(time_t requested_backup_id) if (errno && errno != ENOENT) { - elog(WARNING, "cannot read date directory \"%s\": %s", - date_ent->d_name, strerror(errno)); + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); goto err_proc; } } @@ -311,21 +312,48 @@ catalog_get_backup_list(time_t requested_backup_id) goto err_proc; } - closedir(date_dir); - date_dir = NULL; + closedir(data_dir); + data_dir = NULL; parray_qsort(backups, pgBackupCompareIdDesc); + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + return backups; err_proc: - if (date_dir) - closedir(date_dir); + if (data_dir) + closedir(data_dir); if (backup) pgBackupFree(backup); if (backups) parray_walk(backups, pgBackupFree); parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + return NULL; } @@ -385,15 +413,17 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "#Configuration\n"); fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); - fprintf(out, "stream = %s\n", backup->stream?"true":"false"); - fprintf(out, "compress-alg = %s\n", deparse_compress_alg(compress_alg)); - fprintf(out, "compress-level = %d\n", compress_level); - fprintf(out, "from-replica = %s\n", from_replica?"true":"false"); - + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); if (backup->server_version[0] != '\0') fprintf(out, "server-version = %s\n", backup->server_version); @@ -429,7 +459,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->data_bytes != BYTES_INVALID) fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); - if (backup->data_bytes != BYTES_INVALID) + if (backup->wal_bytes != BYTES_INVALID) fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); fprintf(out, "status = %s\n", status2str(backup->status)); @@ -461,6 +491,30 @@ pgBackupWriteBackupControlFile(pgBackup *backup) fclose(fp); } +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + /* * Read BACKUP_CONTROL_FILE and create pgBackup. * - Comment starts with ';'. @@ -475,10 +529,10 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; - char *compress_alg = NULL; + char *program_version = NULL; char *server_version = NULL; - int *compress_level; - bool *from_replica; + char *compress_alg = NULL; + int parsed_options; pgut_option options[] = { @@ -495,22 +549,41 @@ readBackupControlFile(const char *path) {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, - {'u', 0, "compress-level", &compress_level, SOURCE_FILE_STRICT}, - {'b', 0, "from-replica", &from_replica, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {0} }; if (access(path, F_OK) != 0) + { + elog(WARNING, "control file \"%s\" doesn't exist", path); + pgBackupFree(backup); return NULL; + } - pgBackup_init(backup); - pgut_readopt(path, options, ERROR); + pgBackupInit(backup); + parsed_options = pgut_readopt(path, options, WARNING); + + if (parsed_options == 0) + { + elog(WARNING, "control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } if (backup_mode) { @@ -546,10 +619,12 @@ readBackupControlFile(const char *path) { if (strcmp(status, "OK") == 0) backup->status = BACKUP_STATUS_OK; - else if (strcmp(status, "RUNNING") == 0) - backup->status = BACKUP_STATUS_RUNNING; else if (strcmp(status, "ERROR") == 0) backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; else if (strcmp(status, "DELETING") == 0) backup->status = BACKUP_STATUS_DELETING; else if (strcmp(status, "DELETED") == 0) @@ -571,6 +646,13 @@ readBackupControlFile(const char *path) free(parent_backup); } + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + if (server_version) { StrNCpy(backup->server_version, server_version, @@ -578,6 +660,9 @@ readBackupControlFile(const char *path) pfree(server_version); } + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + return backup; } @@ -626,11 +711,106 @@ deparse_backup_mode(BackupMode mode) return NULL; } +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + /* free pgBackup object */ void pgBackupFree(void *backup) { - free(backup); + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); } /* Compare two pgBackup with their IDs (start time) in ascending order */ @@ -687,3 +867,48 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, make_native_path(path); } + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/configure.c b/src/configure.c index 55d2bbc5..bb67ec5f 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,19 +2,38 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" +#include "utils/logger.h" + +#include "pqexpbuffer.h" + +#include "utils/json.h" + static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + static pgBackupConfig *cur_config = NULL; +static PQExpBufferData show_buf; +static int32 json_level = 0; + +/* + * All this code needs refactoring. + */ + /* Set configure options */ int do_configure(bool show_only) @@ -39,13 +58,17 @@ do_configure(bool show_only) config->master_db = master_db; if (master_user) config->master_user = master_user; - if (replica_timeout != 300) /* 300 is default value */ + + if (replica_timeout) config->replica_timeout = replica_timeout; - if (log_level_console != LOG_NONE) - config->log_level_console = LOG_LEVEL_CONSOLE; - if (log_level_file != LOG_NONE) - config->log_level_file = LOG_LEVEL_FILE; + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; if (log_filename) config->log_filename = log_filename; if (error_log_filename) @@ -62,13 +85,13 @@ do_configure(bool show_only) if (retention_window) config->retention_window = retention_window; - if (compress_alg != NOT_DEFINED_COMPRESS) + if (compress_alg) config->compress_alg = compress_alg; - if (compress_level != DEFAULT_COMPRESS_LEVEL) + if (compress_level) config->compress_level = compress_level; if (show_only) - writeBackupCatalogConfig(stderr, config); + show_configure(config); else writeBackupCatalogConfigFile(config); @@ -89,21 +112,23 @@ pgBackupConfigInit(pgBackupConfig *config) config->master_port = NULL; config->master_db = NULL; config->master_user = NULL; - config->replica_timeout = INT_MIN; /* INT_MIN means "undefined" */ + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - config->log_level_console = INT_MIN; /* INT_MIN means "undefined" */ - config->log_level_file = INT_MIN; /* INT_MIN means "undefined" */ - config->log_filename = NULL; + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; config->error_log_filename = NULL; - config->log_directory = NULL; - config->log_rotation_size = 0; - config->log_rotation_age = 0; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; - config->retention_redundancy = 0; - config->retention_window = 0; + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; - config->compress_alg = NOT_DEFINED_COMPRESS; - config->compress_level = DEFAULT_COMPRESS_LEVEL; + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; } void @@ -114,7 +139,7 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) fprintf(out, "#Backup instance info\n"); fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); fprintf(out, "#Connection parameters:\n"); if (config->pgdatabase) @@ -136,55 +161,43 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) if (config->master_user) fprintf(out, "master-user = %s\n", config->master_user); - if (config->replica_timeout != INT_MIN) - { - convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, - &res, &unit); - fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); - } + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); fprintf(out, "#Logging parameters:\n"); - if (config->log_level_console != INT_MIN) - fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); - if (config->log_level_file != INT_MIN) - fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); - if (config->log_filename) - fprintf(out, "log-filename = %s\n", config->log_filename); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); if (config->error_log_filename) fprintf(out, "error-log-filename = %s\n", config->error_log_filename); - if (config->log_directory) - fprintf(out, "log-directory = %s\n", config->log_directory); - /* - * Convert values from base unit - */ - if (config->log_rotation_size) - { - convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, - &res, &unit); - fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, unit); - } - if (config->log_rotation_age) - { - convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, - &res, &unit); - fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, unit); - } + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); fprintf(out, "#Retention parameters:\n"); - if (config->retention_redundancy) - fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); - if (config->retention_window) - fprintf(out, "retention-window = %u\n", config->retention_window); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); fprintf(out, "#Compression parameters:\n"); fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); - - if (compress_level != config->compress_level) - fprintf(out, "compress-level = %d\n", compress_level); - else - fprintf(out, "compress-level = %d\n", config->compress_level); + fprintf(out, "compress-level = %d\n", config->compress_level); } void @@ -240,6 +253,8 @@ readBackupCatalogConfigFile(void) { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, /* other options */ { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, {0} }; @@ -251,7 +266,6 @@ readBackupCatalogConfigFile(void) pgut_readopt(path, options, ERROR); return config; - } static void @@ -271,3 +285,155 @@ opt_compress_alg(pgut_option *opt, const char *arg) { cur_config->compress_alg = parse_compress_alg(arg); } + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Logging parameters */ + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + { + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); + } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); + + /* Retention parameters */ + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/data.c b/src/data.c index a1bdb91a..a66770bc 100644 --- a/src/data.c +++ b/src/data.c @@ -27,18 +27,24 @@ #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ -static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) { - uLongf compressed_size = dst_size; - int rc = compress2(dst, &compressed_size, src, src_size, compress_level); + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + return rc == Z_OK ? compressed_size : rc; } /* Implementation of zlib compression method */ -static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) { - uLongf dest_len = dst_size; - int rc = uncompress(dst, &dest_len, src, src_size); + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + return rc == Z_OK ? dest_len : rc; } #endif @@ -47,8 +53,9 @@ static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_ * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. */ -static size_t -do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +static int32 +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) { switch (alg) { @@ -57,7 +64,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_compress(dst, dst_size, src, src_size); + return zlib_compress(dst, dst_size, src, src_size, level); #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -70,8 +77,9 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, Compre * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -static size_t -do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +static int32 +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) { switch (alg) { @@ -101,6 +109,7 @@ typedef struct BackupPageHeader /* Special value for compressed_size field */ #define PageIsTruncated -2 +#define SkipCurrentPage -3 /* Verify page's header */ static bool @@ -134,8 +143,8 @@ static int read_page_from_file(pgFile *file, BlockNumber blknum, FILE *in, Page page, XLogRecPtr *page_lsn) { - off_t offset = blknum*BLCKSZ; - size_t read_len = 0; + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; /* read the block */ if (fseek(in, offset, SEEK_SET) != 0) @@ -216,31 +225,32 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* - * Backup the specified block from a file of a relation. - * Verify page header and checksum of the page and write it - * to the backup file. + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated */ -static void -backup_data_page(backup_files_args *arguments, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - BlockNumber blknum, BlockNumber nblocks, - FILE *in, FILE *out, - pg_crc32 *crc, int *n_skipped, - BackupMode backup_mode) +static int32 +prepare_page(backup_files_arg *arguments, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + BlockNumber blknum, BlockNumber nblocks, + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) { - BackupPageHeader header; - Page page = malloc(BLCKSZ); - Page compressed_page = NULL; - XLogRecPtr page_lsn = 0; - size_t write_buffer_size; - char write_buffer[BLCKSZ+sizeof(header)]; - - int try_again = 100; - bool page_is_valid = false; + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; - header.block = blknum; - header.compressed_size = 0; + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); /* * Read the page and verify its header and checksum. @@ -258,7 +268,7 @@ backup_data_page(backup_files_args *arguments, if (result == 0) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; /* Page is not actually valid, but it is absent * and we're not going to reread it or validate */ page_is_valid = true; @@ -291,35 +301,38 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) { size_t page_size = 0; - - free(page); - page = NULL; - page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size); - if (page == NULL) + if (ptrack_page == NULL) { /* This block was truncated.*/ - header.compressed_size = PageIsTruncated; + page_is_truncated = true; } else if (page_size != BLCKSZ) { + free(ptrack_page); elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", file->path, absolute_blknum, BLCKSZ, page_size); } else { /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); if (is_checksum_enabled) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* get lsn from page, provided by pg_ptrack_get_block() */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && !parse_page(page, &page_lsn)) elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " "Possible risk of a memory corruption"); @@ -328,76 +341,91 @@ backup_data_page(backup_files_args *arguments, if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - header.compressed_size != PageIsTruncated && + !page_is_truncated && page_lsn < prev_backup_start_lsn) { elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); (*n_skipped)++; - free(page); + return SkipCurrentPage; + } + + if (page_is_truncated) + return PageIsTruncated; + + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page, + CompressAlg calg, int clevel) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) return; - } - if (header.compressed_size != PageIsTruncated) + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) { - file->read_size += BLCKSZ; - - compressed_page = malloc(BLCKSZ); + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, compress_alg); - - file->compress_alg = compress_alg; + page, BLCKSZ, calg, clevel); + file->compress_alg = calg; + file->read_size += BLCKSZ; Assert (header.compressed_size <= BLCKSZ); - } - write_buffer_size = sizeof(header); - - /* - * The page was truncated. Write only header - * to know that we must truncate restored file - */ - if (header.compressed_size == PageIsTruncated) - { - memcpy(write_buffer, &header, sizeof(header)); - } - /* The page compression failed. Write it as is. */ - else if (header.compressed_size == -1) - { - header.compressed_size = BLCKSZ; - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), page, BLCKSZ); - write_buffer_size += header.compressed_size; - } - /* The page was successfully compressed */ - else if (header.compressed_size > 0) - { - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), compressed_page, header.compressed_size); - write_buffer_size += MAXALIGN(header.compressed_size); + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } } /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", blknum, header.compressed_size, write_buffer_size); */ /* Update CRC */ - COMP_CRC32C(*crc, &write_buffer, write_buffer_size); + COMP_CRC32C(*crc, write_buffer, write_buffer_size); /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { - int errno_tmp = errno; + int errno_tmp = errno; + fclose(in); fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", - file->path, blknum, strerror(errno_tmp)); + file->path, blknum, strerror(errno_tmp)); } file->write_size += write_buffer_size; - - if (page != NULL) - free(page); - if (compressed_page != NULL) - free(compressed_page); } /* @@ -409,18 +437,19 @@ backup_data_page(backup_files_args *arguments, * backup with special header. */ bool -backup_data_file(backup_files_args* arguments, - const char *from_root, const char *to_root, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - BackupMode backup_mode) +backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) { - char to_path[MAXPGPATH]; - FILE *in; - FILE *out; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int n_blocks_skipped = 0; - int n_blocks_read = 0; + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; /* * Skip unchanged file only if it exists in previous backup. @@ -430,7 +459,7 @@ backup_data_file(backup_files_args* arguments, if ((backup_mode == BACKUP_MODE_DIFF_PAGE || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev) + file->exists_in_prev && !file->pagemap_isabsent) { /* * There are no changed blocks since last backup. We want make @@ -446,7 +475,7 @@ backup_data_file(backup_files_args* arguments, INIT_CRC32C(file->crc); /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(file->crc); @@ -479,8 +508,7 @@ backup_data_file(backup_files_args* arguments, nblocks = file->size/BLCKSZ; /* open backup file for write */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -493,32 +521,45 @@ backup_data_file(backup_files_args* arguments, * Read each page, verify checksum and write it to backup. * If page map is empty or file is not present in previous backup * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty - || file->pagemap.bitmapsize == PageBitmapIsAbsent - || !file->exists_in_prev) + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) { for (blknum = 0; blknum < nblocks; blknum++) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); n_blocks_read++; + if (page_state == PageIsTruncated) + break; } if (backup_mode == BACKUP_MODE_DIFF_DELTA) file->n_blocks = n_blocks_read; } - /* If page map is not empty we scan only changed blocks, */ + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ else { datapagemap_iterator_t *iter; iter = datapagemap_iterate(&file->pagemap); while (datapagemap_next(iter, &blknum)) { - backup_data_page(arguments, file, prev_backup_start_lsn, blknum, - nblocks, in, out, &(file->crc), - &n_blocks_skipped, backup_mode); + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); n_blocks_read++; + if (page_state == PageIsTruncated) + break; } pg_free(file->pagemap.bitmap); @@ -562,25 +603,26 @@ backup_data_file(backup_files_args* arguments, /* * Restore files in the from_root directory to the to_root directory with * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. */ void -restore_data_file(const char *from_root, - const char *to_root, - pgFile *file, - pgBackup *backup) +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) { - char to_path[MAXPGPATH]; - FILE *in = NULL; - FILE *out = NULL; - BackupPageHeader header; - BlockNumber blknum; - size_t file_size; + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) { /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { elog(ERROR, "cannot open backup file \"%s\": %s", file->path, @@ -593,10 +635,9 @@ restore_data_file(const char *from_root, * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "r+"); + out = fopen(to_path, PG_BINARY_R "+"); if (out == NULL && errno == ENOENT) - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -605,8 +646,9 @@ restore_data_file(const char *from_root, to_path, strerror(errno_tmp)); } - for (blknum = 0; ; blknum++) + while (true) { + off_t write_pos; size_t read_len; DataPage compressed_page; /* used as read buffer */ DataPage page; @@ -615,6 +657,21 @@ restore_data_file(const char *from_root, if (file->write_size == BYTES_INVALID) break; + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); if (read_len != sizeof(header)) @@ -632,18 +689,19 @@ restore_data_file(const char *from_root, } if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u",file->path, blknum); + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); + + blknum = header.block; if (header.compressed_size == PageIsTruncated) { /* * Backup contains information that this block was truncated. - * Truncate file to this length. + * We need to truncate file to this length. */ - if (ftruncate(fileno(out), header.block * BLCKSZ) != 0) - elog(ERROR, "cannot truncate \"%s\": %s", - file->path, strerror(errno)); - elog(VERBOSE, "truncate file %s to block %u", file->path, header.block); + truncate_from = blknum; + need_truncate = true; break; } @@ -657,64 +715,89 @@ restore_data_file(const char *from_root, if (header.compressed_size != BLCKSZ) { - size_t uncompressed_size = 0; + int32 uncompressed_size = 0; uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, file->compress_alg); + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); if (uncompressed_size != BLCKSZ) - elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", uncompressed_size); + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); } + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + /* * Seek and write the restored page. */ - blknum = header.block; - if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0) + if (fseek(out, write_pos, SEEK_SET) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + if (header.compressed_size < BLCKSZ) { if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, file->path, strerror(errno)); } else { /* if page wasn't compressed, we've read full block */ if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, file->path, strerror(errno)); } } - /* - * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do - * But during DELTA backup we read every file in PGDATA and thus DELTA backup - * knows exact size of every file at the time of backup. - * So when restoring file from DELTA backup we, knowning it`s size at - * a time of a backup, can truncate file to this size. - */ - - if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA) + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) { + size_t file_size = 0; + /* get file current size */ fseek(out, 0, SEEK_END); file_size = ftell(out); + if (file_size > file->n_blocks * BLCKSZ) { - /* - * Truncate file to this length. - */ - if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0) - elog(ERROR, "cannot truncate \"%s\": %s", - file->path, strerror(errno)); - elog(INFO, "Delta truncate file %s to block %u", file->path, file->n_blocks); + truncate_from = file->n_blocks; + need_truncate = true; } } + if (need_truncate) + { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), write_pos) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + /* update file permission */ if (chmod(to_path, file->mode) == -1) { @@ -759,7 +842,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); @@ -775,7 +858,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, "w"); + out = fopen(to_path, PG_BINARY_W); if (out == NULL) { int errno_tmp = errno; @@ -843,7 +926,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->read_size += read_len; } - file->write_size = file->read_size; + file->write_size = (int64) file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; @@ -867,6 +950,22 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + #ifdef HAVE_LIBZ /* * Show error during work with compressed file @@ -918,7 +1017,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out; + FILE *out=NULL; char buf[XLOG_BLCKSZ]; const char *to_path_p = to_path; char to_path_temp[MAXPGPATH]; @@ -930,7 +1029,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -946,7 +1045,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = gzopen(to_path_temp, "wb"); + gz_out = gzopen(to_path_temp, PG_BINARY_W); if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", compress_level, to_path_temp, get_gz_error(gz_out, errno)); @@ -961,7 +1060,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1083,7 +1182,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open file for read */ - in = fopen(from_path, "r"); + in = fopen(from_path, PG_BINARY_R); if (in == NULL) { #ifdef HAVE_LIBZ @@ -1092,7 +1191,7 @@ get_wal_file(const char *from_path, const char *to_path) * extension. */ snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - gz_in = gzopen(gz_from_path, "rb"); + gz_in = gzopen(gz_from_path, PG_BINARY_R); if (gz_in == NULL) { if (errno == ENOENT) @@ -1120,7 +1219,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, "w"); + out = fopen(to_path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1254,7 +1353,7 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, "r"); + in = fopen(file->path, PG_BINARY_R); if (in == NULL) { FIN_CRC32C(crc); diff --git a/src/delete.c b/src/delete.c index f81fe70d..0829e725 100644 --- a/src/delete.c +++ b/src/delete.c @@ -33,8 +33,6 @@ do_delete(time_t backup_id) /* Get complete list of backups */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); if (backup_id != 0) { @@ -141,7 +139,8 @@ do_retention_purge(void) if (retention_window > 0) elog(LOG, "WINDOW=%u", retention_window); - if (retention_redundancy == 0 && retention_window == 0) + if (retention_redundancy == 0 + && retention_window == 0) { elog(WARNING, "Retention policy is not set"); if (!delete_wal) @@ -161,7 +160,8 @@ do_retention_purge(void) } /* Find target backups to be deleted */ - if (delete_expired && (retention_redundancy > 0 || retention_window > 0)) + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) { backup_num = 0; for (i = 0; i < parray_num(backup_list); i++) @@ -173,13 +173,13 @@ do_retention_purge(void) if (backup->status != BACKUP_STATUS_OK) continue; /* - * When a validate full backup was found, we can delete the + * When a valid full backup was found, we can delete the * backup that is older than it using the number of generations. */ if (backup->backup_mode == BACKUP_MODE_FULL) backup_num++; - /* Evaluateretention_redundancy if this backup is eligible for removal */ + /* Evaluate retention_redundancy if this backup is eligible for removal */ if (keep_next_backup || retention_redundancy >= backup_num_evaluate + 1 || (retention_window > 0 && backup->recovery_time >= days_threshold)) @@ -200,6 +200,7 @@ do_retention_purge(void) continue; } + /* Delete backup and update status to DELETED */ pgBackupDeleteFiles(backup); backup_deleted = true; @@ -259,7 +260,8 @@ pgBackupDeleteFiles(pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - elog(INFO, "delete: %s %s", base36enc(backup->start_time), timestamp); + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); /* * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which @@ -281,7 +283,7 @@ pgBackupDeleteFiles(pgBackup *backup) /* print progress */ elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, - (unsigned long) parray_num(files), file->path); + (unsigned long) parray_num(files), file->path); if (remove(file->path)) { diff --git a/src/dir.c b/src/dir.c index 8df3da3f..a08bd934 100644 --- a/src/dir.c +++ b/src/dir.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include #include #include @@ -88,6 +87,34 @@ static char *pgdata_exclude_files_non_exclusive[] = NULL }; +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); @@ -95,17 +122,23 @@ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list); +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + /* * Create directory, also create parent directories if necessary. */ int dir_create_dir(const char *dir, mode_t mode) { - char copy[MAXPGPATH]; char parent[MAXPGPATH]; - strncpy(copy, dir, MAXPGPATH); - strncpy(parent, dirname(copy), MAXPGPATH); + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); /* Create parent first */ if (access(parent, F_OK) == -1) @@ -153,6 +186,8 @@ pgFileInit(const char *path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); + file->name = NULL; + file->size = 0; file->mode = 0; file->read_size = 0; @@ -161,7 +196,8 @@ pgFileInit(const char *path) file->is_datafile = false; file->linked = NULL; file->pagemap.bitmap = NULL; - file->pagemap.bitmapsize = PageBitmapIsAbsent; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; file->tblspcOid = 0; file->dbOid = 0; file->relOid = 0; @@ -185,7 +221,8 @@ pgFileInit(const char *path) file->is_cfs = false; file->exists_in_prev = false; /* can change only in Incremental backup. */ - file->n_blocks = -1; /* can change only in DELTA backup. Number of blocks readed during backup */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; return file; } @@ -223,7 +260,7 @@ delete_file: } pg_crc32 -pgFileGetCRC(pgFile *file) +pgFileGetCRC(const char *file_path) { FILE *fp; pg_crc32 crc = 0; @@ -232,10 +269,10 @@ pgFileGetCRC(pgFile *file) int errno_tmp; /* open file in binary read mode */ - fp = fopen(file->path, "r"); + fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", - file->path, strerror(errno)); + file_path, strerror(errno)); /* calc CRC of backup file */ INIT_CRC32C(crc); @@ -247,7 +284,7 @@ pgFileGetCRC(pgFile *file) } errno_tmp = errno; if (!feof(fp)) - elog(WARNING, "cannot read \"%s\": %s", file->path, + elog(WARNING, "cannot read \"%s\": %s", file_path, strerror(errno_tmp)); if (len > 0) COMP_CRC32C(crc, buf, len); @@ -350,7 +387,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, "r"); + black_list_file = fopen(path, PG_BINARY_R); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); @@ -385,7 +422,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); - parray_qsort(files, pgFileComparePath); } /* @@ -676,7 +712,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, * **is_root** is a little bit hack. We exclude only first level of directories * and on the first level we check all files and directories. */ -void +static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude) { @@ -748,6 +784,277 @@ list_data_directories(parray *files, const char *path, bool is_root, path, strerror(prev_errno)); } +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + /* * Read names of symbolik names of tablespaces with links to directories from * tablespace_map or tablespace_map.txt. @@ -799,6 +1106,70 @@ read_tablespace_map(parray *files, const char *backup_dir) fclose(fp); } +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + /* * Print backup content list. */ @@ -817,20 +1188,25 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," - "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\"", - path, (unsigned long) file->write_size, file->mode, - file->is_datafile?1:0, file->is_cfs?1:0, file->crc, + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg)); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); +#ifndef WIN32 if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif fprintf(out, ",\"linked\":\"%s\"", file->linked); - if (file->n_blocks != -1) + if (file->n_blocks != BLOCKNUM_INVALID) fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); fprintf(out, "}\n"); @@ -852,23 +1228,25 @@ print_file_list(FILE *out, const parray *files, const char *root) * {"name1":"value1", "name2":"value2"} * * The value will be returned to "value_str" as string if it is not NULL. If it - * is NULL the value will be returned to "value_ulong" as unsigned long. + * is NULL the value will be returned to "value_int64" as int64. + * + * Returns true if the value was found in the line. */ -static void +static bool get_control_value(const char *str, const char *name, - char *value_str, uint64 *value_uint64, bool is_mandatory) + char *value_str, int64 *value_int64, bool is_mandatory) { int state = CONTROL_WAIT_NAME; char *name_ptr = (char *) name; char *buf = (char *) str; - char buf_uint64[32], /* Buffer for "value_uint64" */ - *buf_uint64_ptr = buf_uint64; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; /* Set default values */ if (value_str) *value_str = '\0'; - else if (value_uint64) - *value_uint64 = 0; + else if (value_int64) + *value_int64 = 0; while (*buf) { @@ -903,7 +1281,7 @@ get_control_value(const char *str, const char *name, if (*buf == '"') { state = CONTROL_INVALUE; - buf_uint64_ptr = buf_uint64; + buf_int64_ptr = buf_int64; } else if (IsAlpha(*buf)) goto bad_format; @@ -916,19 +1294,19 @@ get_control_value(const char *str, const char *name, { *value_str = '\0'; } - else if (value_uint64) + else if (value_int64) { /* Length of buf_uint64 should not be greater than 31 */ - if (buf_uint64_ptr - buf_uint64 >= 32) + if (buf_int64_ptr - buf_int64 >= 32) elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", name, str, DATABASE_FILE_LIST); - *buf_uint64_ptr = '\0'; - if (!parse_uint64(buf_uint64, value_uint64, 0)) + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) goto bad_format; } - return; + return true; } else { @@ -939,8 +1317,8 @@ get_control_value(const char *str, const char *name, } else { - *buf_uint64_ptr = *buf; - buf_uint64_ptr++; + *buf_int64_ptr = *buf; + buf_int64_ptr++; } } break; @@ -964,11 +1342,12 @@ get_control_value(const char *str, const char *name, if (is_mandatory) elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", name, str, DATABASE_FILE_LIST); - return; + return false; bad_format: elog(ERROR, "%s file has invalid format in line %s", DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ } /* @@ -995,7 +1374,7 @@ dir_read_file_list(const char *root, const char *file_txt) char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; - uint64 write_size, + int64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, @@ -1010,12 +1389,7 @@ dir_read_file_list(const char *root, const char *file_txt) get_control_value(buf, "is_datafile", NULL, &is_datafile, true); get_control_value(buf, "is_cfs", NULL, &is_cfs, false); get_control_value(buf, "crc", NULL, &crc, true); - - /* optional fields */ - get_control_value(buf, "linked", linked, NULL, false); - get_control_value(buf, "segno", NULL, &segno, false); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "n_blocks", NULL, &n_blocks, false); if (root) join_path_components(filepath, root, path); @@ -1024,16 +1398,25 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); - file->write_size = (size_t) write_size; + file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; file->crc = (pg_crc32) crc; file->compress_alg = parse_compress_alg(compress_alg_string); - if (linked[0]) + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) file->linked = pgut_strdup(linked); - file->segno = (int) segno; - file->n_blocks = (int) n_blocks; + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; parray_append(files, file); } @@ -1095,3 +1478,14 @@ fileExists(const char *path) else return true; } + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/help.c b/src/help.c index 2f84b225..9ad6cc2e 100644 --- a/src/help.c +++ b/src/help.c @@ -14,6 +14,7 @@ static void help_restore(void); static void help_validate(void); static void help_show(void); static void help_delete(void); +static void help_merge(void); static void help_set_config(void); static void help_show_config(void); static void help_add_instance(void); @@ -36,6 +37,8 @@ help_command(char *command) help_show(); else if (strcmp(command, "delete") == 0) help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); else if (strcmp(command, "set-config") == 0) help_set_config(); else if (strcmp(command, "show-config") == 0) @@ -56,7 +59,7 @@ help_command(char *command) || strcmp(command, "-V") == 0) printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); else - printf(_("Unknown command. Try pg_probackup help\n")); + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); exit(0); } @@ -87,8 +90,10 @@ help_pg_probackup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); @@ -111,26 +116,31 @@ help_pg_probackup(void) printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--replica-timeout=timeout]\n")); printf(_("\n %s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); printf(_("\n %s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); @@ -253,7 +263,7 @@ help_backup(void) printf(_(" --master-db=db_name database to connect to master\n")); printf(_(" --master-host=host_name database server host of master\n")); printf(_(" --master-port=port database server port of master\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication in seconds\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); } static void @@ -261,11 +271,11 @@ help_restore(void) { printf(_("%s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica]\n\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -276,6 +286,7 @@ help_restore(void) printf(_(" --progress show progress\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); @@ -290,6 +301,7 @@ help_restore(void) printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -318,7 +330,7 @@ help_validate(void) { printf(_("%s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -328,8 +340,11 @@ help_validate(void) printf(_(" --progress show progress\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -357,11 +372,13 @@ static void help_show(void) { printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n\n")); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void @@ -399,6 +416,48 @@ help_delete(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } +static void +help_merge(void) +{ + printf(_("%s merge -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceed this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'KB', 'MB', 'GB', 'TB' (default: KB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceed this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + static void help_set_config(void) { @@ -418,6 +477,7 @@ help_set_config(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -466,16 +526,20 @@ help_set_config(void) printf(_(" --master-db=db_name database to connect to master\n")); printf(_(" --master-host=host_name database server host of master\n")); printf(_(" --master-port=port database server port of master\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); } static void help_show_config(void) { - printf(_("%s show-config -B backup-dir --instance=instance_name\n\n"), PROGRAM_NAME); + printf(_("%s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); } static void diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 00000000..b149ef5b --- /dev/null +++ b/src/merge.c @@ -0,0 +1,525 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + to_backup->wal_bytes = XLOG_SEG_SIZE * + (to_backup->stop_lsn / XLogSegSize - to_backup->start_lsn / XLogSegSize + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i, + num_files = parray_num(argument->files); + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/parsexlog.c b/src/parsexlog.c index a0079b42..3e5e8790 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,24 +5,23 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ -#include "postgres_fe.h" - #include "pg_probackup.h" +#include #include +#ifdef HAVE_LIBZ +#include +#endif #include "commands/dbcommands_xlog.h" #include "catalog/storage_xlog.h" #include "access/transam.h" - -#ifdef HAVE_LIBZ -#include -#endif +#include "utils/thread.h" /* * RmgrNames is an array of resource manager names, to make error messages @@ -85,74 +84,134 @@ typedef struct xl_xact_abort static void extractPageInfo(XLogReaderState *record); static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); -static int xlogreadfd = -1; -static XLogSegNo xlogreadsegno = -1; -static char xlogfpath[MAXPGPATH]; -static bool xlogexists = false; - -#ifdef HAVE_LIBZ -static gzFile gz_xlogread = NULL; -static char gz_xlogfpath[MAXPGPATH]; -#endif - typedef struct XLogPageReadPrivate { const char *archivedir; TimeLineID tli; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif } XLogPageReadPrivate; +/* An argument for a thread function */ +typedef struct +{ + int thread_num; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* - * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the - * given timeline. Collect data blocks touched by the WAL records into a page map. - * - * If **prev_segno** is true then read all segments up to **endpoint** segment - * minus one. Else read all segments up to **endpoint** segment. + * extractPageMap() worker. */ -void -extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, - XLogRecPtr endpoint, bool prev_segno, parray *files) +static void * +doExtractPageMap(void *arg) { - size_t i; - XLogRecord *record; + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; char *errormsg; - XLogPageReadPrivate private; - XLogSegNo endSegNo, - nextSegNo = 0; - elog(LOG, "Compiling pagemap"); - if (!XRecOffIsValid(startpoint)) - elog(ERROR, "Invalid startpoint value %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - - if (!XRecOffIsValid(endpoint)) - elog(ERROR, "Invalid endpoint value %X/%X", - (uint32) (endpoint >> 32), (uint32) (endpoint)); - - private.archivedir = archivedir; - private.tli = tli; - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); + private_data = &extract_arg->private_data; + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); if (xlogreader == NULL) elog(ERROR, "out of memory"); - XLByteToSeg(endpoint, endSegNo); - if (prev_segno) - endSegNo--; + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; do { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; - errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) + { + private_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + private_data->xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (private_data->xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + XLogSegNoOffsetToRecPtr(private_data->xlogsegno, 0, + extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; if (errormsg) elog(WARNING, "could not read WAL record at %X/%X: %s", @@ -167,37 +226,127 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, * start_lsn, we won't be able to build page map and PAGE backup will * be incorrect. Stop it and throw an error. */ - if (!xlogexists) - elog(ERROR, "WAL segment \"%s\" is absent", xlogfpath); - else if (xlogreadfd != -1) - elog(ERROR, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); + PrintXLogCorruptionMsg(private_data, ERROR); } extractPageInfo(xlogreader); - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; XLByteToSeg(xlogreader->EndRecPtr, nextSegNo); - } while (nextSegNo <= endSegNo && xlogreader->EndRecPtr != endpoint); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, + XLogRecPtr endpoint, bool prev_seg, parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t *threads; + xlog_thread_arg *thread_args; + time_t start_time, + end_time; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + XLByteToSeg(endpoint, endSegNo); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + time(&start_time); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, false); + thread_args[i].thread_num = i; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + XLByteToSeg(startpoint, nextSegNoToRead); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; + + threads_need++; } - /* Mark every datafile with empty pagemap as unchanged */ - for (i = 0; i < parray_num(files); i++) + /* Run threads */ + for (i = 0; i < threads_need; i++) { - pgFile *file = (pgFile *) parray_get(files, i); - if (file->is_datafile && file->pagemap.bitmap == NULL) - file->pagemap.bitmapsize = PageBitmapIsEmpty; + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); } - elog(LOG, "Pagemap compiled"); + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + + pfree(threads); + pfree(thread_args); + + time(&end_time); + if (extract_isok) + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap compiling failed"); } /* @@ -205,8 +354,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, */ static void validate_backup_wal_from_start_to_stop(pgBackup *backup, - char *backup_xlog_path, - TimeLineID tli) + char *backup_xlog_path, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; XLogRecord *record; @@ -215,15 +363,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, XLogPageReadPrivate private; bool got_endpoint = false; - private.archivedir = backup_xlog_path; - private.tli = tli; - - /* We will check it in the end */ - xlogfpath[0] = '\0'; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, true); while (true) { @@ -248,45 +388,27 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, if (!got_endpoint) { - if (xlogfpath[0] != 0) - { - /* XLOG reader couldn't read WAL segment. - * We throw a WARNING here to be able to update backup status below. - */ - if (!xlogexists) - { - elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); - } - else if (xlogreadfd != -1) - { - elog(WARNING, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); - } - } + PrintXLogCorruptionMsg(&private, WARNING); /* * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - backup->status = BACKUP_STATUS_CORRUPT; - pgBackupWriteBackupControlFile(backup); - elog(WARNING, "There are not enough WAL records to consistenly restore " - "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", - base36enc(backup->start_time), - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn), - (uint32) (backup->stop_lsn >> 32), - (uint32) (backup->stop_lsn)); + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); } /* clean */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } } /* @@ -299,6 +421,7 @@ validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; @@ -350,7 +473,7 @@ validate_wal(pgBackup *backup, * If recovery target is provided check that we can restore backup to a * recovery target time or xid. */ - if (!TransactionIdIsValid(target_xid) && target_time == 0) + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ elog(INFO, "Backup %s WAL segments are valid", backup_id); @@ -369,22 +492,15 @@ validate_wal(pgBackup *backup, * up to the given recovery target. * In any case we cannot restore to the point before stop_lsn. */ - private.archivedir = archivedir; - - private.tli = tli; - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); - - /* We will check it in the end */ - xlogfpath[0] = '\0'; + xlogreader = InitXLogPageRead(&private, archivedir, tli, true); /* We can restore at least up to the backup end */ time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); last_xid = backup->recovery_xid; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) - || (target_time != 0 && backup->recovery_time >= target_time)) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) all_wal = true; startpoint = backup->stop_lsn; @@ -439,21 +555,7 @@ validate_wal(pgBackup *backup, /* Some needed WAL records are absent */ else { - if (xlogfpath[0] != 0) - { - /* XLOG reader couldn't read WAL segment. - * We throw a WARNING here to be able to update backup status below. - */ - if (!xlogexists) - { - elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); - } - else if (xlogreadfd != -1) - { - elog(WARNING, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); - } - } + PrintXLogCorruptionMsg(&private, WARNING); elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, last_timestamp, last_xid); @@ -470,16 +572,14 @@ validate_wal(pgBackup *backup, else if (target_time != 0) elog(ERROR, "not enough WAL records to time %s", target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } /* clean */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } } /* @@ -505,12 +605,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - private.archivedir = archivedir; - private.tli = tli; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, archivedir, tli, true); /* Read records from stop_lsn down to start_lsn */ do @@ -553,13 +648,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, res = false; cleanup: + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } return res; } @@ -581,23 +671,13 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "Invalid target_lsn value %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - private.archivedir = archivedir; - private.tli = target_tli; - - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - if (xlogreader == NULL) - elog(ERROR, "out of memory"); + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, true); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - if (xlogreadfd != -1) - { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; - } return res; } @@ -626,54 +706,53 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogPageReadPrivate *private_data; uint32 targetPageOff; + private_data = (XLogPageReadPrivate *) xlogreader->private_data; targetPageOff = targetPagePtr % XLogSegSize; /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (!XLByteInSeg(targetPagePtr, xlogreadsegno)) + if (!XLByteInSeg(targetPagePtr, private_data->xlogsegno)) { - if (xlogreadfd >= 0) + CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ + if (private_data->manual_switch) { - close(xlogreadfd); - xlogreadfd = -1; - xlogexists = false; + private_data->need_switch = true; + return -1; } -#ifdef HAVE_LIBZ - else if (gz_xlogread != NULL) - { - gzclose(gz_xlogread); - gz_xlogread = NULL; - xlogexists = false; - } -#endif } - XLByteToSeg(targetPagePtr, xlogreadsegno); + XLByteToSeg(targetPagePtr, private_data->xlogsegno); - if (!xlogexists) + /* Try to switch to the next WAL segment */ + if (!private_data->xlogexists) { char xlogfname[MAXFNAMELEN]; - XLogFileName(xlogfname, private->tli, xlogreadsegno); - snprintf(xlogfpath, MAXPGPATH, "%s/%s", private->archivedir, - xlogfname); + XLogFileName(xlogfname, private_data->tli, private_data->xlogsegno); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); - if (fileExists(xlogfpath)) + if (fileExists(private_data->xlogpath)) { - elog(LOG, "Opening WAL segment \"%s\"", xlogfpath); + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); - xlogexists = true; - xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); - if (xlogreadfd < 0) + if (private_data->xlogfile < 0) { elog(WARNING, "Could not open WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } } @@ -681,17 +760,21 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* Try to open compressed WAL segment */ else { - snprintf(gz_xlogfpath, sizeof(gz_xlogfpath), "%s.gz", xlogfpath); - if (fileExists(gz_xlogfpath)) + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) { - elog(LOG, "Opening compressed WAL segment \"%s\"", gz_xlogfpath); + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); - xlogexists = true; - gz_xlogread = gzopen(gz_xlogfpath, "rb"); - if (gz_xlogread == NULL) + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) { elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", - gz_xlogfpath, strerror(errno)); + private_data->gz_xlogpath, strerror(errno)); return -1; } } @@ -699,55 +782,129 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #endif /* Exit without error if WAL segment doesn't exist */ - if (!xlogexists) + if (!private_data->xlogexists) return -1; } /* * At this point, we have the right segment open. */ - Assert(xlogexists); + Assert(private_data->xlogexists); /* Read the requested page */ - if (xlogreadfd != -1) + if (private_data->xlogfile != -1) { - if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) { elog(WARNING, "Could not seek in WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } - if (read(xlogreadfd, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Could not read from WAL segment \"%s\": %s", - xlogfpath, strerror(errno)); + private_data->xlogpath, strerror(errno)); return -1; } } #ifdef HAVE_LIBZ else { - if (gzseek(gz_xlogread, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", - gz_xlogfpath, get_gz_error(gz_xlogread)); + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); return -1; } - if (gzread(gz_xlogread, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", - gz_xlogfpath, get_gz_error(gz_xlogread)); + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); return -1; } } #endif - *pageTLI = private->tli; + *pageTLI = private_data->tli; return XLOG_BLCKSZ; } +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlogfile = -1; + + if (allocate_reader) + { + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogpath != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + /* * Extract information about blocks modified in this record. */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5d464171..a18ee5c3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -10,14 +10,16 @@ #include "pg_probackup.h" #include "streamutil.h" +#include "utils/thread.h" #include #include #include #include #include +#include "pg_getopt.h" -const char *PROGRAM_VERSION = "2.0.17"; +const char *PROGRAM_VERSION = "2.0.18"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; @@ -36,7 +38,7 @@ char backup_instance_path[MAXPGPATH]; char arclog_path[MAXPGPATH] = ""; /* common options */ -char *backup_id_string_param = NULL; +static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; bool progress = false; @@ -47,28 +49,29 @@ char *replication_slot = NULL; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -bool from_replica = false; bool is_remote_backup = false; /* Wait timeout for WAL segment archiving */ -uint32 archive_timeout = 300; /* default is 300 seconds */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; const char *master_db = NULL; const char *master_host = NULL; const char *master_port= NULL; const char *master_user = NULL; -uint32 replica_timeout = 300; /* default is 300 seconds */ +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; /* restore options */ static char *target_time; static char *target_xid; +static char *target_lsn; static char *target_inclusive; static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; -static char *target_action = NULL;; +static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; bool restore_as_replica = false; +bool restore_no_validate = false; /* delete options */ bool delete_wal = false; @@ -81,10 +84,11 @@ uint32 retention_redundancy = 0; uint32 retention_window = 0; /* compression options */ -CompressAlg compress_alg = NOT_DEFINED_COMPRESS; -int compress_level = DEFAULT_COMPRESS_LEVEL; +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; bool compress_shortcut = false; + /* other options */ char *instance_name; uint64 system_identifier = 0; @@ -94,30 +98,34 @@ static char *wal_file_path; static char *wal_file_name; static bool file_overwrite = false; +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + /* current settings */ pgBackup current; -ProbackupSubcmd backup_subcmd; +ProbackupSubcmd backup_subcmd = NO_CMD; -bool help = false; +static bool help_opt = false; static void opt_backup_mode(pgut_option *opt, const char *arg); static void opt_log_level_console(pgut_option *opt, const char *arg); static void opt_log_level_file(pgut_option *opt, const char *arg); static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); static void compress_init(void); static pgut_option options[] = { /* directory options */ - { 'b', 1, "help", &help, SOURCE_CMDLINE }, + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, /* common options */ { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, - { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, /* backup options */ { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, @@ -143,12 +151,14 @@ static pgut_option options[] = { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, /* delete options */ { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, /* TODO not implemented yet */ - { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, /* retention options */ { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, @@ -178,6 +188,8 @@ static pgut_option options[] = { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, { 0 } }; @@ -187,14 +199,14 @@ static pgut_option options[] = int main(int argc, char *argv[]) { - char *command = NULL; - char path[MAXPGPATH]; + char *command = NULL, + *command_name; /* Check if backup_path is directory. */ struct stat stat_buf; int rc; /* initialize configuration */ - pgBackup_init(¤t); + pgBackupInit(¤t); PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); @@ -202,42 +214,40 @@ main(int argc, char *argv[]) /* * Save main thread's tid. It is used call exit() in case of errors. */ -#ifdef WIN32 - main_tid = GetCurrentThreadId(); -#else main_tid = pthread_self(); -#endif /* Parse subcommands and non-subcommand options */ if (argc > 1) { if (strcmp(argv[1], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH; + backup_subcmd = ARCHIVE_PUSH_CMD; else if (strcmp(argv[1], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET; + backup_subcmd = ARCHIVE_GET_CMD; else if (strcmp(argv[1], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE; + backup_subcmd = ADD_INSTANCE_CMD; else if (strcmp(argv[1], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE; + backup_subcmd = DELETE_INSTANCE_CMD; else if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT; + backup_subcmd = INIT_CMD; else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP; + backup_subcmd = BACKUP_CMD; else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE; + backup_subcmd = RESTORE_CMD; else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW; + backup_subcmd = VALIDATE_CMD; else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG; + backup_subcmd = SET_CONFIG_CMD; else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG; - else if (strcmp(argv[1], "--help") == 0 - || strcmp(argv[1], "help") == 0 - || strcmp(argv[1], "-?") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) { if (argc > 2) help_command(argv[2]); @@ -248,35 +258,33 @@ main(int argc, char *argv[]) || strcmp(argv[1], "version") == 0 || strcmp(argv[1], "-V") == 0) { - if (argc == 2) - { #ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); #else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); #endif - exit(0); - } - else if (strcmp(argv[2], "--help") == 0) - help_command(argv[1]); - else - elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); + exit(0); } else - elog(ERROR, "Unknown subcommand"); + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); } + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ - if (backup_subcmd == BACKUP || - backup_subcmd == RESTORE || - backup_subcmd == VALIDATE || - backup_subcmd == DELETE) + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) { int i, len = 0, @@ -303,11 +311,12 @@ main(int argc, char *argv[]) command[len] = '\0'; } + optind += 1; /* Parse command line arguments */ pgut_getopt(argc, argv, options); - if (help) - help_command(argv[2]); + if (help_opt) + help_command(command_name); /* backup_path is required for all pg_probackup commands except help */ if (backup_path == NULL) @@ -320,6 +329,7 @@ main(int argc, char *argv[]) if (backup_path == NULL) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } + canonicalize_path(backup_path); /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) @@ -340,7 +350,8 @@ main(int argc, char *argv[]) } /* Option --instance is required for all commands except init and show */ - if (backup_subcmd != INIT && backup_subcmd != SHOW && backup_subcmd != VALIDATE) + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) { if (instance_name == NULL) elog(ERROR, "required parameter not specified: --instance"); @@ -352,7 +363,8 @@ main(int argc, char *argv[]) */ if (instance_name) { - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); /* @@ -360,7 +372,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT && backup_subcmd != ADD_INSTANCE) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -372,8 +384,10 @@ main(int argc, char *argv[]) * Read options from env variables or from config file, * unless we're going to set them via set-config. */ - if (instance_name && backup_subcmd != SET_CONFIG) + if (instance_name && backup_subcmd != SET_CONFIG_CMD) { + char path[MAXPGPATH]; + /* Read environment variables */ pgut_getopt_env(options); @@ -393,18 +407,19 @@ main(int argc, char *argv[]) elog(ERROR, "-D, --pgdata must be an absolute path"); /* Sanity check of --backup-id option */ - if (backup_id_string_param != NULL) + if (backup_id_string != NULL) { - if (backup_subcmd != RESTORE - && backup_subcmd != VALIDATE - && backup_subcmd != DELETE - && backup_subcmd != SHOW) - elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", - argv[1]); + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); - current.backup_id = base36dec(backup_id_string_param); + current.backup_id = base36dec(backup_id_string); if (current.backup_id == 0) - elog(ERROR, "Invalid backup-id"); + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); } /* Setup stream options. They are used in streamutil.c. */ @@ -426,12 +441,12 @@ main(int argc, char *argv[]) pgdata_exclude_dir[i] = "pg_log"; } - if (backup_subcmd == VALIDATE || backup_subcmd == RESTORE) + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_immediate, - target_name, target_action); + target_inclusive, target_tli, target_lsn, target_immediate, + target_name, target_action, restore_no_validate); } if (num_threads < 1) @@ -442,23 +457,24 @@ main(int argc, char *argv[]) /* do actual operation */ switch (backup_subcmd) { - case ARCHIVE_PUSH: + case ARCHIVE_PUSH_CMD: return do_archive_push(wal_file_path, wal_file_name, file_overwrite); - case ARCHIVE_GET: + case ARCHIVE_GET_CMD: return do_archive_get(wal_file_path, wal_file_name); - case ADD_INSTANCE: + case ADD_INSTANCE_CMD: return do_add_instance(); - case DELETE_INSTANCE: + case DELETE_INSTANCE_CMD: return do_delete_instance(); - case INIT: + case INIT_CMD: return do_init(); - case BACKUP: + case BACKUP_CMD: { const char *backup_mode; time_t start_time; start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, @@ -466,34 +482,40 @@ main(int argc, char *argv[]) return do_backup(start_time); } - case RESTORE: + case RESTORE_CMD: return do_restore_or_validate(current.backup_id, recovery_target_options, true); - case VALIDATE: + case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0) return do_validate_all(); else return do_restore_or_validate(current.backup_id, recovery_target_options, false); - case SHOW: + case SHOW_CMD: return do_show(current.backup_id); - case DELETE: - if (delete_expired && backup_id_string_param) + case DELETE_CMD: + if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); - if (!delete_expired && !delete_wal && !backup_id_string_param) + if (!delete_expired && !delete_wal && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); - if (delete_wal && !delete_expired && !backup_id_string_param) + if (delete_wal && !delete_expired && !backup_id_string) return do_retention_purge(); if (delete_expired) return do_retention_purge(); else return do_delete(current.backup_id); - case SHOW_CONFIG: + case MERGE_CMD: + do_merge(current.backup_id); + break; + case SHOW_CONFIG_CMD: return do_configure(true); - case SET_CONFIG: + case SET_CONFIG_CMD: return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); } return 0; @@ -517,49 +539,31 @@ opt_log_level_file(pgut_option *opt, const char *arg) log_level_file = parse_log_level(arg); } -CompressAlg -parse_compress_alg(const char *arg) +static void +opt_show_format(pgut_option *opt, const char *arg) { + const char *v = arg; size_t len; /* Skip all spaces detected */ - while (isspace((unsigned char)*arg)) - arg++; - len = strlen(arg); + while (IsSpace(*v)) + v++; + len = strlen(v); - if (len == 0) - elog(ERROR, "compress algrorithm is empty"); - - if (pg_strncasecmp("zlib", arg, len) == 0) - return ZLIB_COMPRESS; - else if (pg_strncasecmp("pglz", arg, len) == 0) - return PGLZ_COMPRESS; - else if (pg_strncasecmp("none", arg, len) == 0) - return NONE_COMPRESS; - else - elog(ERROR, "invalid compress algorithm value \"%s\"", arg); - - return NOT_DEFINED_COMPRESS; -} - -const char* -deparse_compress_alg(int alg) -{ - switch (alg) + if (len > 0) { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - return "none"; - case ZLIB_COMPRESS: - return "zlib"; - case PGLZ_COMPRESS: - return "pglz"; + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } - - return NULL; + else + elog(ERROR, "Invalid show format \"%s\"", arg); } -void +static void opt_compress_alg(pgut_option *opt, const char *arg) { compress_alg = parse_compress_alg(arg); @@ -568,17 +572,17 @@ opt_compress_alg(pgut_option *opt, const char *arg) /* * Initialize compress and sanity checks for compress. */ -static -void compress_init(void) +static void +compress_init(void) { /* Default algorithm is zlib */ if (compress_shortcut) compress_alg = ZLIB_COMPRESS; - if (backup_subcmd != SET_CONFIG) + if (backup_subcmd != SET_CONFIG_CMD) { - if (compress_level != DEFAULT_COMPRESS_LEVEL - && compress_alg == NONE_COMPRESS) + if (compress_level != COMPRESS_LEVEL_DEFAULT + && compress_alg == NOT_DEFINED_COMPRESS) elog(ERROR, "Cannot specify compress-level option without compress-alg option"); } @@ -588,7 +592,7 @@ void compress_init(void) if (compress_level == 0) compress_alg = NOT_DEFINED_COMPRESS; - if (backup_subcmd == BACKUP || backup_subcmd == ARCHIVE_PUSH) + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ if (compress_alg == ZLIB_COMPRESS) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ce..9da22ad6 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -15,10 +15,6 @@ #include #include -#ifndef WIN32 -#include -#endif - #include "access/timeline.h" #include "access/xlogdefs.h" #include "access/xlog_internal.h" @@ -28,6 +24,13 @@ #include "storage/checksum.h" #include "utils/pg_crc.h" #include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif #include "utils/parray.h" #include "utils/pgut.h" @@ -60,6 +63,8 @@ #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" /* Direcotry/File permission */ #define DIR_PERMISSION (0700) #define FILE_PERMISSION (0600) @@ -85,9 +90,10 @@ typedef struct pgFile size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - size_t write_size; /* size of the backed-up file. BYTES_INVALID means + int64 write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ + /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ @@ -102,21 +108,23 @@ typedef struct pgFile bool is_database; bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile uint32 lock; /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ } pgFile; /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ -#define PageBitmapIsAbsent -1 /* Used to mark files with unknown state of pagemap, i.e. datafiles without _ptrack */ /* Current state of backup */ typedef enum BackupStatus { BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ BACKUP_STATUS_OK, /* completed backup */ - BACKUP_STATUS_RUNNING, /* running backup */ BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ BACKUP_STATUS_DELETING, /* data files are being deleted */ BACKUP_STATUS_DELETED, /* data files have been deleted */ BACKUP_STATUS_DONE, /* completed but not validated yet */ @@ -135,24 +143,33 @@ typedef enum BackupMode typedef enum ProbackupSubcmd { - INIT = 0, - ARCHIVE_PUSH, - ARCHIVE_GET, - ADD_INSTANCE, - DELETE_INSTANCE, - BACKUP, - RESTORE, - VALIDATE, - SHOW, - DELETE, - SET_CONFIG, - SHOW_CONFIG + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD } ProbackupSubcmd; +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + /* special values of pgBackup fields */ -#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) typedef struct pgBackupConfig { @@ -169,6 +186,8 @@ typedef struct pgBackupConfig const char *master_user; int replica_timeout; + int archive_timeout; + int log_level_console; int log_level_file; char *log_filename; @@ -184,6 +203,8 @@ typedef struct pgBackupConfig int compress_level; } pgBackupConfig; +typedef struct pgBackup pgBackup; + /* Information about single backup stored in backup.conf */ typedef struct pgBackup { @@ -214,18 +235,24 @@ typedef struct pgBackup /* Size of WAL files in archive needed to restore this backup */ int64 wal_bytes; + CompressAlg compress_alg; + int compress_level; + /* Fields needed for compatibility check */ uint32 block_size; uint32 wal_block_size; uint32 checksum_version; + char program_version[100]; char server_version[100]; - bool stream; /* Was this backup taken in stream mode? + bool stream; /* Was this backup taken in stream mode? * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ time_t parent_backup; /* Identifier of the previous backup. * Which is basic backup for this * incremental backup. */ + pgBackup *parent_backup_link; char *primary_conninfo; /* Connection parameters of the backup * in the format suitable for recovery.conf */ } pgBackup; @@ -241,12 +268,17 @@ typedef struct pgRecoveryTarget TransactionId recovery_target_xid; /* add one more field in order to avoid deparsing recovery_target_xid back */ const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; TimeLineID recovery_target_tli; bool recovery_target_inclusive; bool inclusive_specified; bool recovery_target_immediate; const char *recovery_target_name; const char *recovery_target_action; + bool restore_no_validate; } pgRecoveryTarget; /* Union to ease operations on relation pages */ @@ -260,18 +292,20 @@ typedef struct { const char *from_root; const char *to_root; - parray *backup_files_list; - parray *prev_backup_filelist; - XLogRecPtr prev_backup_start_lsn; - PGconn *thread_backup_conn; - PGcancel *thread_cancel_conn; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. */ int ret; -} backup_files_args; +} backup_files_arg; /* * return pointer that exceeds the length of prefix from character string. @@ -309,13 +343,14 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 extern uint32 archive_timeout; -extern bool from_replica; extern bool is_remote_backup; extern const char *master_db; extern const char *master_host; extern const char *master_port; extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 extern uint32 replica_timeout; extern bool is_ptrack_support; @@ -331,7 +366,10 @@ extern bool delete_expired; extern bool apply_to_all; extern bool force_delete; -/* retention options */ +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + extern uint32 retention_redundancy; extern uint32 retention_window; @@ -340,7 +378,8 @@ extern CompressAlg compress_alg; extern int compress_level; extern bool compress_shortcut; -#define DEFAULT_COMPRESS_LEVEL 6 +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); @@ -348,9 +387,12 @@ extern const char* deparse_compress_alg(int alg); extern char *instance_name; extern uint64 system_identifier; +/* show options */ +extern ShowFormat show_format; + /* current settings */ extern pgBackup current; -extern ProbackupSubcmd backup_subcmd; +extern ProbackupSubcmd backup_subcmd; /* in dir.c */ /* exclude directory list for $PGDATA file listing */ @@ -363,7 +405,7 @@ extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern char *pg_ptrack_get_block(backup_files_args *arguments, +extern char *pg_ptrack_get_block(backup_files_arg *arguments, Oid dbOid, Oid tblsOid, Oid relOid, BlockNumber blknum, size_t *result_size); @@ -377,10 +419,12 @@ extern bool satisfy_recovery_target(const pgBackup *backup, extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, - const char *target_inclusive, TimeLineID target_tli, bool target_immediate, - const char *target_name, const char *target_action); + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); -extern void opt_tablespace_map(pgut_option *opt, const char *arg); +/* in merge.c */ +extern void do_merge(time_t backup_id); /* in init.c */ extern int do_init(void); @@ -432,21 +476,31 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); +extern pgBackup* find_parent_backup(pgBackup *current_backup); + /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root); -extern void list_data_directories(parray *files, const char *path, - bool is_root, bool exclude); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); extern parray *dir_read_file_list(const char *root, const char *file_txt); @@ -455,26 +509,29 @@ extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, bool omit_symlink); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(pgFile *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); /* in data.c */ -extern bool backup_data_file(backup_files_args* arguments, - const char *from_root, const char *to_root, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - BackupMode backup_mode); -extern void restore_data_file(const char *from_root, const char *to_root, - pgFile *file, pgBackup *backup); -extern bool copy_file(const char *from_root, const char *to_root, - pgFile *file); +extern bool backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); @@ -485,12 +542,13 @@ extern bool calc_file_checksum(pgFile *file); extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli, - XLogRecPtr endpoint, bool prev_segno, + XLogRecPtr endpoint, bool prev_seg, parray *backup_files_list); extern void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, @@ -513,9 +571,21 @@ extern long unsigned int base36dec(const char *text); extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern pg_time_t timestamptz_to_time_t(TimestampTz t); -extern void pgBackup_init(pgBackup *backup); +extern int parse_server_version(char *server_version_str); /* in status.c */ extern bool is_pg_running(void); +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + #endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c index 69bac841..3fa8f09f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -17,76 +17,38 @@ #include #include "catalog/pg_control.h" +#include "utils/logger.h" +#include "utils/thread.h" typedef struct { - parray *files; - pgBackup *backup; + parray *files; + pgBackup *backup; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. */ int ret; -} restore_files_args; - -/* Tablespace mapping structures */ - -typedef struct TablespaceListCell -{ - struct TablespaceListCell *next; - char old_dir[MAXPGPATH]; - char new_dir[MAXPGPATH]; -} TablespaceListCell; - -typedef struct TablespaceList -{ - TablespaceListCell *head; - TablespaceListCell *tail; -} TablespaceList; - -typedef struct TablespaceCreatedListCell -{ - struct TablespaceCreatedListCell *next; - char link_name[MAXPGPATH]; - char linked_dir[MAXPGPATH]; -} TablespaceCreatedListCell; - -typedef struct TablespaceCreatedList -{ - TablespaceCreatedListCell *head; - TablespaceCreatedListCell *tail; -} TablespaceCreatedList; +} restore_files_arg; static void restore_backup(pgBackup *backup); -static void restore_directories(const char *pg_data_dir, - const char *backup_dir); -static void check_tablespace_mapping(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); -static void restore_files(void *arg); +static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); -static const char *get_tablespace_mapping(const char *dir); -static void set_tablespace_created(const char *link, const char *dir); -static const char *get_tablespace_created(const char *link); - -/* Tablespace mapping */ -static TablespaceList tablespace_dirs = {NULL, NULL}; -static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int -do_restore_or_validate(time_t target_backup_id, - pgRecoveryTarget *rt, - bool is_restore) +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) { - int i; + int i = 0; parray *backups; - parray *timelines; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; @@ -94,7 +56,7 @@ do_restore_or_validate(time_t target_backup_id, int dest_backup_index = 0; int base_full_backup_index = 0; int corrupted_backup_index = 0; - char *action = is_restore ? "Restore":"Validate"; + char *action = is_restore ? "Restore":"Validate"; if (is_restore) { @@ -115,13 +77,12 @@ do_restore_or_validate(time_t target_backup_id, catalog_lock(); /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backups == NULL) - elog(ERROR, "Failed to get backup list."); /* Find backup range we should restore or validate. */ - for (i = 0; i < parray_num(backups); i++) + while ((i < parray_num(backups)) && !dest_backup) { current_backup = (pgBackup *) parray_get(backups, i); + i++; /* Skip all backups which started after target backup */ if (target_backup_id && current_backup->start_time > target_backup_id) @@ -133,7 +94,6 @@ do_restore_or_validate(time_t target_backup_id, */ if (is_restore && - !dest_backup && target_backup_id == INVALID_BACKUP_ID && current_backup->status != BACKUP_STATUS_OK) { @@ -147,8 +107,7 @@ do_restore_or_validate(time_t target_backup_id, * ensure that it satisfies recovery target. */ if ((target_backup_id == current_backup->start_time - || target_backup_id == INVALID_BACKUP_ID) - && !dest_backup) + || target_backup_id == INVALID_BACKUP_ID)) { /* backup is not ok, @@ -169,6 +128,8 @@ do_restore_or_validate(time_t target_backup_id, if (rt->recovery_target_tli) { + parray *timelines; + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); /* Read timeline history files from archives */ timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); @@ -199,37 +160,42 @@ do_restore_or_validate(time_t target_backup_id, * Save it as dest_backup */ dest_backup = current_backup; - dest_backup_index = i; + dest_backup_index = i-1; + } + } + + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); } - /* If we already found dest_backup, look for full backup. */ - if (dest_backup) + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) { - if (current_backup->backup_mode == BACKUP_MODE_FULL) + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) { - if (current_backup->status != BACKUP_STATUS_OK) - { - /* Full backup revalidation can be done only for DONE and CORRUPT */ - if (current_backup->status == BACKUP_STATUS_DONE || - current_backup->status == BACKUP_STATUS_CORRUPT) - elog(WARNING, "base backup %s for given backup %s is in %s status, trying to revalidate", - base36enc_dup(current_backup->start_time), - base36enc_dup(dest_backup->start_time), - status2str(current_backup->status)); - else - elog(ERROR, "base backup %s for given backup %s is in %s status", - base36enc_dup(current_backup->start_time), - base36enc_dup(dest_backup->start_time), - status2str(current_backup->status)); - } - /* We found both dest and base backups. */ - base_full_backup = current_backup; base_full_backup_index = i; break; } - else - /* It`s ok to skip incremental backup */ - continue; } } @@ -243,66 +209,71 @@ do_restore_or_validate(time_t target_backup_id, if (is_restore) check_tablespace_mapping(dest_backup); - if (dest_backup->backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); - - /* - * Validate backups from base_full_backup to dest_backup. - */ - for (i = base_full_backup_index; i >= dest_backup_index; i--) + if (!is_restore || !rt->restore_no_validate) { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - pgBackupValidate(backup); - /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ - if (backup->status == BACKUP_STATUS_CORRUPT) - { - corrupted_backup = backup; - corrupted_backup_index = i; - break; - } - /* We do not validate WAL files of intermediate backups - * It`s done to speed up restore - */ - } - /* There is no point in wal validation - * if there is corrupted backup between base_backup and dest_backup - */ - if (!corrupted_backup) + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + /* - * Validate corresponding WAL files. - * We pass base_full_backup timeline as last argument to this function, - * because it's needed to form the name of xlog file. + * Validate backups from base_full_backup to dest_backup. */ - validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, base_full_backup->tli); - - /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ - if (corrupted_backup) - { - for (i = corrupted_backup_index - 1; i >= 0; i--) + for (i = base_full_backup_index; i >= dest_backup_index; i--) { pgBackup *backup = (pgBackup *) parray_get(backups, i); - /* Mark incremental OK backup as orphan */ - if (backup->backup_mode == BACKUP_MODE_FULL) - break; - if (backup->status != BACKUP_STATUS_OK) - continue; - else + + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) { - char *backup_id, - *corrupted_backup_id; + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli); - backup->status = BACKUP_STATUS_ORPHAN; - pgBackupWriteBackupControlFile(backup); + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; - backup_id = base36enc_dup(backup->start_time); - corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); - elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", - backup_id, corrupted_backup_id); + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); - free(backup_id); - free(corrupted_backup_id); + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } } } } @@ -312,7 +283,12 @@ do_restore_or_validate(time_t target_backup_id, * produce corresponding error message */ if (dest_backup->status == BACKUP_STATUS_OK) - elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } else if (dest_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); else if (dest_backup->status == BACKUP_STATUS_ORPHAN) @@ -327,6 +303,11 @@ do_restore_or_validate(time_t target_backup_id, for (i = base_full_backup_index; i >= dest_backup_index; i--) { pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + restore_backup(backup); } @@ -362,8 +343,9 @@ restore_backup(pgBackup *backup) char list_path[MAXPGPATH]; parray *files; int i; - pthread_t restore_threads[num_threads]; - restore_files_args *restore_threads_args[num_threads]; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg *threads_args; bool restore_isok = true; if (backup->status != BACKUP_STATUS_OK) @@ -388,7 +370,7 @@ restore_backup(pgBackup *backup) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - restore_directories(pgdata, this_backup_path); + create_data_directories(pgdata, this_backup_path, true); /* * Get list of files which need to be restored. @@ -397,46 +379,50 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); files = dir_read_file_list(database_path, list_path); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_clear_flag(&file->lock); } /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { - restore_files_args *arg = pg_malloc(sizeof(restore_files_args)); + restore_files_arg *arg = &(threads_args[i]); + arg->files = files; arg->backup = backup; /* By default there are some error */ - arg->ret = 1; + threads_args[i].ret = 1; elog(LOG, "Start thread for num:%li", parray_num(files)); - restore_threads_args[i] = arg; - pthread_create(&restore_threads[i], NULL, - (void *(*)(void *)) restore_files, arg); + pthread_create(&threads[i], NULL, restore_files, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { - pthread_join(restore_threads[i], NULL); - if (restore_threads_args[i]->ret == 1) + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) restore_isok = false; - - pg_free(restore_threads_args[i]); } if (!restore_isok) elog(ERROR, "Data files restoring failed"); + pfree(threads); + pfree(threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) + if (log_level_console <= LOG || log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); } @@ -452,7 +438,7 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - int i; + int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ @@ -473,7 +459,7 @@ remove_deleted_files(pgBackup *backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) + if (log_level_console <= LOG || log_level_file <= LOG) elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); } } @@ -485,228 +471,14 @@ remove_deleted_files(pgBackup *backup) parray_free(files_restored); } -/* - * Restore backup directories from **backup_database_dir** to **pg_data_dir**. - * - * TODO: Think about simplification and clarity of the function. - */ -static void -restore_directories(const char *pg_data_dir, const char *backup_dir) -{ - parray *dirs, - *links; - size_t i; - char backup_database_dir[MAXPGPATH], - to_path[MAXPGPATH]; - - dirs = parray_new(); - links = parray_new(); - - join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); - - list_data_directories(dirs, backup_database_dir, true, false); - read_tablespace_map(links, backup_dir); - - elog(LOG, "restore directories and symlinks..."); - - for (i = 0; i < parray_num(dirs); i++) - { - pgFile *dir = (pgFile *) parray_get(dirs, i); - char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); - - Assert(S_ISDIR(dir->mode)); - - /* First try to create symlink and linked directory */ - if (path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) - { - char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), - *link_sep, - *tmp_ptr; - char link_name[MAXPGPATH]; - pgFile **link; - - /* Extract link name from relative path */ - link_sep = first_dir_separator(link_ptr); - if (link_sep != NULL) - { - int len = link_sep - link_ptr; - strncpy(link_name, link_ptr, len); - link_name[len] = '\0'; - } - else - goto create_directory; - - tmp_ptr = dir->path; - dir->path = link_name; - /* Search only by symlink name without path */ - link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); - dir->path = tmp_ptr; - - if (link) - { - const char *linked_path = get_tablespace_mapping((*link)->linked); - const char *dir_created; - - if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - linked_path); - - /* Check if linked directory was created earlier */ - dir_created = get_tablespace_created(link_name); - if (dir_created) - { - /* - * If symlink and linked directory were created do not - * create it second time. - */ - if (strcmp(dir_created, linked_path) == 0) - { - /* - * Create rest of directories. - * First check is there any directory name after - * separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - else - continue; - } - else - elog(ERROR, "tablespace directory \"%s\" of page backup does not " - "match with previous created tablespace directory \"%s\" of symlink \"%s\"", - linked_path, dir_created, link_name); - } - - /* - * This check was done in check_tablespace_mapping(). But do - * it again. - */ - if (!dir_is_empty(linked_path)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - - if (link_sep) - elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", - linked_path, - (int) (link_sep - relative_ptr), relative_ptr); - else - elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", - linked_path, relative_ptr); - - /* Firstly, create linked directory */ - dir_create_dir(linked_path, DIR_PERMISSION); - - join_path_components(to_path, pg_data_dir, PG_TBLSPC_DIR); - /* Create pg_tblspc directory just in case */ - dir_create_dir(to_path, DIR_PERMISSION); - - /* Secondly, create link */ - join_path_components(to_path, to_path, link_name); - if (symlink(linked_path, to_path) < 0) - elog(ERROR, "could not create symbolic link \"%s\": %s", - to_path, strerror(errno)); - - /* Save linked directory */ - set_tablespace_created(link_name, linked_path); - - /* - * Create rest of directories. - * First check is there any directory name after separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - - continue; - } - } - -create_directory: - elog(LOG, "create directory \"%s\"", relative_ptr); - - /* This is not symlink, create directory */ - join_path_components(to_path, pg_data_dir, relative_ptr); - dir_create_dir(to_path, DIR_PERMISSION); - } - - parray_walk(links, pgFileFree); - parray_free(links); - - parray_walk(dirs, pgFileFree); - parray_free(dirs); -} - -/* - * Check that all tablespace mapping entries have correct linked directory - * paths. Linked directories must be empty or do not exist. - * - * If tablespace-mapping option is supplied, all OLDDIR entries must have - * entries in tablespace_map file. - */ -static void -check_tablespace_mapping(pgBackup *backup) -{ - char this_backup_path[MAXPGPATH]; - parray *links; - size_t i; - TablespaceListCell *cell; - pgFile *tmp_file = pgut_new(pgFile); - - links = parray_new(); - - pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - read_tablespace_map(links, this_backup_path); - - if (LOG_LEVEL_CONSOLE <= LOG || LOG_LEVEL_FILE <= LOG) - elog(LOG, "check tablespace directories of backup %s", - base36enc(backup->start_time)); - - /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ - for (cell = tablespace_dirs.head; cell; cell = cell->next) - { - tmp_file->linked = cell->old_dir; - - if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) - elog(ERROR, "--tablespace-mapping option's old directory " - "doesn't have an entry in tablespace_map file: \"%s\"", - cell->old_dir); - } - - /* 2 - all linked directories must be empty */ - for (i = 0; i < parray_num(links); i++) - { - pgFile *link = (pgFile *) parray_get(links, i); - const char *linked_path = link->linked; - TablespaceListCell *cell; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - if (strcmp(link->linked, cell->old_dir) == 0) - { - linked_path = cell->new_dir; - break; - } - - if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - linked_path); - - if (!dir_is_empty(linked_path)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - } - - free(tmp_file); - parray_walk(links, pgFileFree); - parray_free(links); -} - /* * Restore files into $PGDATA. */ -static void +static void * restore_files(void *arg) { int i; - restore_files_args *arguments = (restore_files_args *)arg; + restore_files_arg *arguments = (restore_files_arg *)arg; for (i = 0; i < parray_num(arguments->files); i++) { @@ -714,7 +486,7 @@ restore_files(void *arg) char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; pgBackupGetPath(arguments->backup, from_root, @@ -730,7 +502,6 @@ restore_files(void *arg) elog(LOG, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), rel_path); - /* * For PAGE and PTRACK backups skip files which haven't changed * since previous backup and thus were not backed up. @@ -741,11 +512,11 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(VERBOSE, "The file didn`t changed. Skip restore: %s", file->path); + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); continue; } - /* Directories was created before */ + /* Directories were created before */ if (S_ISDIR(file->mode)) { elog(VERBOSE, "directory, skip"); @@ -765,20 +536,31 @@ restore_files(void *arg) * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) - restore_data_file(from_root, pgdata, file, arguments->backup); + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } else copy_file(from_root, pgdata, file); /* print size of restored file */ if (file->write_size != BYTES_INVALID) - elog(LOG, "Restored file %s : %lu bytes", - file->path, (unsigned long) file->write_size); + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", + file->path, file->write_size); } /* Data files restoring is successful */ arguments->ret = 0; + + return NULL; } /* Create recovery.conf with given recovery target parameters */ @@ -787,9 +569,9 @@ create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup) { - char path[MAXPGPATH]; - FILE *fp; - bool need_restore_conf = false; + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; if (!backup->stream || (rt->time_specified || rt->xid_specified)) @@ -831,6 +613,9 @@ create_recovery_conf(time_t backup_id, if (rt->xid_specified) fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + if (rt->recovery_target_immediate) fprintf(fp, "recovery_target = 'immediate'\n"); @@ -959,7 +744,8 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - entry->end = (uint32) (-1UL << 32) | -1UL; + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; parray_insert(result, 0, entry); return result; @@ -974,16 +760,22 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) if (rt->time_specified) return backup->recovery_time <= rt->recovery_target_time; + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + return true; } bool satisfy_timeline(const parray *timelines, const pgBackup *backup) { - int i; + int i; + for (i = 0; i < parray_num(timelines); i++) { - TimeLineHistoryEntry *timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && backup->stop_lsn < timeline->end) return true; @@ -999,33 +791,40 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, + const char *target_lsn, bool target_immediate, const char *target_name, - const char *target_action) + const char *target_action, + bool restore_no_validate) { - time_t dummy_time; - TransactionId dummy_xid; - bool dummy_bool; + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; + XLogRecPtr dummy_lsn; /* * count the number of the mutually exclusive options which may specify * recovery target. If final value > 1, throw an error. */ - int recovery_target_specified = 0; + int recovery_target_specified = 0; pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); /* fill all options with default values */ rt->time_specified = false; rt->xid_specified = false; rt->inclusive_specified = false; + rt->lsn_specified = false; rt->recovery_target_time = 0; rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; rt->target_time_string = NULL; rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; rt->recovery_target_inclusive = false; rt->recovery_target_tli = 0; rt->recovery_target_immediate = false; rt->recovery_target_name = NULL; rt->recovery_target_action = NULL; + rt->restore_no_validate = false; /* parse given options */ if (target_time) @@ -1034,7 +833,7 @@ parseRecoveryTargetOptions(const char *target_time, rt->time_specified = true; rt->target_time_string = target_time; - if (parse_time(target_time, &dummy_time)) + if (parse_time(target_time, &dummy_time, false)) rt->recovery_target_time = dummy_time; else elog(ERROR, "Invalid value of --time option %s", target_time); @@ -1056,6 +855,17 @@ parseRecoveryTargetOptions(const char *target_time, elog(ERROR, "Invalid value of --xid option %s", target_xid); } + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + if (target_inclusive) { rt->inclusive_specified = true; @@ -1072,6 +882,11 @@ parseRecoveryTargetOptions(const char *target_time, rt->recovery_target_immediate = target_immediate; } + if (restore_no_validate) + { + rt->restore_no_validate = restore_no_validate; + } + if (target_name) { recovery_target_specified++; @@ -1095,123 +910,11 @@ parseRecoveryTargetOptions(const char *target_time, /* More than one mutually exclusive option was defined. */ if (recovery_target_specified > 1) - elog(ERROR, "At most one of --immediate, --target-name, --time, or --xid can be used"); + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); /* If none of the options is defined, '--inclusive' option is meaningless */ - if (!(rt->xid_specified || rt->time_specified) && rt->recovery_target_inclusive) + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); return rt; } - -/* - * Split argument into old_dir and new_dir and append to tablespace mapping - * list. - * - * Copy of function tablespace_list_append() from pg_basebackup.c. - */ -void -opt_tablespace_map(pgut_option *opt, const char *arg) -{ - TablespaceListCell *cell = pgut_new(TablespaceListCell); - char *dst; - char *dst_ptr; - const char *arg_ptr; - - dst_ptr = dst = cell->old_dir; - for (arg_ptr = arg; *arg_ptr; arg_ptr++) - { - if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "directory name too long"); - - if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') - ; /* skip backslash escaping = */ - else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) - { - if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); - else - dst = dst_ptr = cell->new_dir; - } - else - *dst_ptr++ = *arg_ptr; - } - - if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid tablespace mapping format \"%s\", " - "must be \"OLDDIR=NEWDIR\"", arg); - - /* - * This check isn't absolutely necessary. But all tablespaces are created - * with absolute directories, so specifying a non-absolute path here would - * just never match, possibly confusing users. It's also good to be - * consistent with the new_dir check. - */ - if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", - cell->old_dir); - - if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", - cell->new_dir); - - if (tablespace_dirs.tail) - tablespace_dirs.tail->next = cell; - else - tablespace_dirs.head = cell; - tablespace_dirs.tail = cell; -} - -/* - * Retrieve tablespace path, either relocated or original depending on whether - * -T was passed or not. - * - * Copy of function get_tablespace_mapping() from pg_basebackup.c. - */ -static const char * -get_tablespace_mapping(const char *dir) -{ - TablespaceListCell *cell; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - if (strcmp(dir, cell->old_dir) == 0) - return cell->new_dir; - - return dir; -} - -/* - * Save create directory path into memory. We can use it in next page restore to - * not raise the error "restore tablespace destination is not empty" in - * restore_directories(). - */ -static void -set_tablespace_created(const char *link, const char *dir) -{ - TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); - - strcpy(cell->link_name, link); - strcpy(cell->linked_dir, dir); - cell->next = NULL; - - if (tablespace_created_dirs.tail) - tablespace_created_dirs.tail->next = cell; - else - tablespace_created_dirs.head = cell; - tablespace_created_dirs.tail = cell; -} - -/* - * Is directory was created when symlink was created in restore_directories(). - */ -static const char * -get_tablespace_created(const char *link) -{ - TablespaceCreatedListCell *cell; - - for (cell = tablespace_created_dirs.head; cell; cell = cell->next) - if (strcmp(link, cell->link_name) == 0) - return cell->linked_dir; - - return NULL; -} diff --git a/src/show.c b/src/show.c index b6eee867..f240ce93 100644 --- a/src/show.c +++ b/src/show.c @@ -3,28 +3,40 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" + #include #include #include #include +#include "pqexpbuffer.h" -static void show_backup_list(FILE *out, parray *backup_list); -static void show_backup_detail(FILE *out, pgBackup *backup); -static int do_show_instance(time_t requested_backup_id); +#include "utils/json.h" + + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static int32 json_level = 0; int do_show(time_t requested_backup_id) { - - if (instance_name == NULL - && requested_backup_id != INVALID_BACKUP_ID) + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use --backup_id option"); if (instance_name == NULL) @@ -38,10 +50,12 @@ do_show(time_t requested_backup_id) join_path_components(path, backup_path, BACKUPS_DIR); dir = opendir(path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); - errno = 0; - while ((dent = readdir(dir))) + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) { char child[MAXPGPATH]; struct stat st; @@ -54,73 +68,47 @@ do_show(time_t requested_backup_id) join_path_components(child, path, dent->d_name); if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; instance_name = dent->d_name; sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - fprintf(stdout, "\nBACKUP INSTANCE '%s'\n", instance_name); - do_show_instance(0); + + show_instance(INVALID_BACKUP_ID, true); } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + return 0; } else - return do_show_instance(requested_backup_id); -} - -/* - * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information - * about all backups in the backup instance. - * If valid backup id is passed, show detailed meta information - * about specified backup. - */ -static int -do_show_instance(time_t requested_backup_id) -{ - if (requested_backup_id != INVALID_BACKUP_ID) - { - pgBackup *backup; - - backup = read_backup(requested_backup_id); - if (backup == NULL) - { - elog(INFO, "Requested backup \"%s\" is not found.", - /* We do not need free base36enc's result, we exit anyway */ - base36enc(requested_backup_id)); - /* This is not error */ - return 0; - } - - show_backup_detail(stdout, backup); - - /* cleanup */ - pgBackupFree(backup); - - } - else - { - parray *backup_list; - - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_list == NULL) - elog(ERROR, "Failed to get backup list."); - - show_backup_list(stdout, backup_list); - - /* cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } - - return 0; + return show_backup(requested_backup_id); } static void pretty_size(int64 size, char *buf, size_t len) { - int exp = 0; + int exp = 0; /* minus means the size is invalid */ if (size < 0) @@ -219,16 +207,111 @@ get_parent_tli(TimeLineID child_tli) return result; } +/* + * Initialize instance visualization. + */ static void -show_backup_list(FILE *out, parray *backup_list) +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) { int i; + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + /* if you add new fields here, fix the header */ /* show header */ - fputs("============================================================================================================================================\n", out); - fputs(" Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); - fputs("============================================================================================================================================\n", out); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); for (i = 0; i < parray_num(backup_list); i++) { @@ -255,27 +338,163 @@ show_backup_list(FILE *out, parray *backup_list) /* Get parent timeline before printing */ parent_tli = get_parent_tli(backup->tli); - fprintf(out, " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", - instance_name, - (backup->server_version[0] ? backup->server_version : "----"), - base36enc(backup->start_time), - timestamp, - pgBackupGetBackupMode(backup), - backup->stream ? "STREAM": "ARCHIVE", - backup->tli, - parent_tli, - duration, - data_bytes_str, - (uint32) (backup->start_lsn >> 32), - (uint32) backup->start_lsn, - (uint32) (backup->stop_lsn >> 32), - (uint32) backup->stop_lsn, - status2str(backup->status)); + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); } } +/* + * Json output. + */ + +/* + * Show instance backups in json format. + */ static void -show_backup_detail(FILE *out, pgBackup *backup) +show_instance_json(parray *backup_list) { - pgBackupWriteControl(out, backup); + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level, true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level, true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level, true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level, true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; } diff --git a/src/status.c b/src/status.c index 1c7c6038..155a07f4 100644 --- a/src/status.c +++ b/src/status.c @@ -38,7 +38,7 @@ get_pgpid(void) snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); - pidf = fopen(pid_file, "r"); + pidf = fopen(pid_file, PG_BINARY_R); if (pidf == NULL) { /* No pid file, not an error on startup */ diff --git a/src/util.c b/src/util.c index f2c84f6e..cb8f9bf6 100644 --- a/src/util.c +++ b/src/util.c @@ -176,8 +176,8 @@ uint32 get_data_checksum_version(bool safe) { ControlFileData ControlFile; - char *buffer; - size_t size; + char *buffer; + size_t size; /* First fetch file... */ buffer = slurpFile(pgdata, "global/pg_control", &size, safe); @@ -191,7 +191,7 @@ get_data_checksum_version(bool safe) /* - * Convert time_t value to ISO-8601 format string + * Convert time_t value to ISO-8601 format string. Always set timezone offset. */ void time2iso(char *buf, size_t len, time_t time) @@ -199,25 +199,23 @@ time2iso(char *buf, size_t len, time_t time) struct tm *ptm = gmtime(&time); time_t gmt = mktime(ptm); time_t offset; + char *ptr = buf; ptm = localtime(&time); offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - strftime(buf, len, "%Y-%m-%d %H:%M:%S", ptm); + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - if (offset != 0) + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) { - buf += strlen(buf); - sprintf(buf, "%c%02d", - (offset >= 0) ? '+' : '-', - abs((int) offset) / SECS_PER_HOUR); - - if (abs((int) offset) % SECS_PER_HOUR != 0) - { - buf += strlen(buf); - sprintf(buf, ":%02d", - abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); - } + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } } @@ -237,6 +235,35 @@ timestamptz_to_time_t(TimestampTz t) return result; } +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + const char * status2str(BackupStatus status) { @@ -244,8 +271,9 @@ status2str(BackupStatus status) { "UNKNOWN", "OK", - "RUNNING", "ERROR", + "RUNNING", + "MERGING", "DELETING", "DELETED", "DONE", @@ -295,26 +323,3 @@ remove_not_digit(char *buf, size_t len, const char *str) } buf[j] = '\0'; } - -/* Fill pgBackup struct with default values */ -void -pgBackup_init(pgBackup *backup) -{ - backup->backup_id = INVALID_BACKUP_ID; - backup->backup_mode = BACKUP_MODE_INVALID; - backup->status = BACKUP_STATUS_INVALID; - backup->tli = 0; - backup->start_lsn = 0; - backup->stop_lsn = 0; - backup->start_time = (time_t) 0; - backup->end_time = (time_t) 0; - backup->recovery_xid = 0; - backup->recovery_time = (time_t) 0; - backup->data_bytes = BYTES_INVALID; - backup->block_size = BLCKSZ; - backup->wal_block_size = XLOG_BLCKSZ; - backup->stream = false; - backup->parent_backup = 0; - backup->primary_conninfo = NULL; - backup->server_version[0] = '\0'; -} diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 00000000..3afbe9e7 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 00000000..cf5a7064 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 0a4f835b..88ee9b6b 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -16,11 +15,13 @@ #include "logger.h" #include "pgut.h" +#include "pg_probackup.h" +#include "thread.h" /* Logger parameters */ -int log_level_console = LOG_NONE; -int log_level_file = LOG_NONE; +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; char *log_filename = NULL; char *error_log_filename = NULL; @@ -74,12 +75,12 @@ void init_logger(const char *root_path) { /* Set log path */ - if (LOG_LEVEL_FILE != LOG_OFF || error_log_filename) + if (log_level_file != LOG_OFF || error_log_filename) { if (log_directory) strcpy(log_path, log_directory); else - join_path_components(log_path, root_path, "log"); + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); } } @@ -138,11 +139,10 @@ exit_if_necessary(int elevel) } /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) #ifdef WIN32 - if (main_tid != GetCurrentThreadId()) ExitThread(elevel); #else - if (!pthread_equal(main_tid, pthread_self())) pthread_exit(NULL); #endif else @@ -166,15 +166,16 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) time_t log_time = (time_t) time(NULL); char strfbuf[128]; - write_to_file = elevel >= LOG_LEVEL_FILE && log_path[0] != '\0'; + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; write_to_error_log = elevel >= ERROR && error_log_filename && log_path[0] != '\0'; - write_to_stderr = elevel >= LOG_LEVEL_CONSOLE && !file_only; + write_to_stderr = elevel >= log_level_console && !file_only; - /* - * There is no need to lock if this is elog() from upper elog(). - */ - pthread_mutex_lock(&log_file_mutex); + pthread_lock(&log_file_mutex); +#ifdef WIN32 + std_args = NULL; + error_args = NULL; +#endif loggin_in_progress = true; /* We need copy args only if we need write to error log file */ @@ -201,7 +202,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (log_file == NULL) { if (log_filename == NULL) - open_logfile(&log_file, "pg_probackup.log"); + open_logfile(&log_file, LOG_FILENAME_DEFAULT); else open_logfile(&log_file, log_filename); } @@ -241,7 +242,6 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_stderr) { write_elevel(stderr, elevel); - if (write_to_file) vfprintf(stderr, fmt, std_args); else @@ -272,7 +272,7 @@ elog_stderr(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < ERROR) + if (elevel < log_level_console && elevel < ERROR) return; va_start(args, fmt); @@ -299,7 +299,7 @@ elog(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -319,7 +319,7 @@ elog_file(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -360,7 +360,7 @@ pg_log(eLogType type, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < LOG_LEVEL_CONSOLE && elevel < LOG_LEVEL_FILE && elevel < ERROR) + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) return; va_start(args, fmt); diff --git a/src/utils/logger.h b/src/utils/logger.h index 315c337a..0177c551 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -36,11 +36,13 @@ extern char *error_log_filename; extern char *log_directory; extern char log_path[MAXPGPATH]; +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 extern int log_rotation_size; extern int log_rotation_age; -#define LOG_LEVEL_CONSOLE ((log_level_console == LOG_NONE) ? INFO : log_level_console) -#define LOG_LEVEL_FILE ((log_level_file == LOG_NONE) ? LOG_OFF : log_level_file) +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF #undef elog extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 56263cfa..37f249a2 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -31,6 +31,7 @@ #define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ #define SECS_PER_MINUTE 60 #define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 const char *PROGRAM_NAME = NULL; @@ -42,12 +43,6 @@ static char *password = NULL; bool prompt_password = true; bool force_password = false; -#ifdef WIN32 -DWORD main_tid = 0; -#else -pthread_t main_tid = 0; -#endif - /* Database connections */ static PGcancel *volatile cancel_conn = NULL; @@ -141,8 +136,10 @@ static const unit_conversion time_unit_conversion_table[] = static size_t option_length(const pgut_option opts[]) { - size_t len; + size_t len; + for (len = 0; opts && opts[len].type; len++) { } + return len; } @@ -162,7 +159,7 @@ option_has_arg(char type) static void option_copy(struct option dst[], const pgut_option opts[], size_t len) { - size_t i; + size_t i; for (i = 0; i < len; i++) { @@ -260,7 +257,8 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) message = "a valid string. But provided: "; break; case 't': - if (parse_time(optarg, opt->var)) + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) return; message = "a time"; break; @@ -750,9 +748,12 @@ parse_uint64(const char *value, uint64 *result, int flags) /* * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. */ bool -parse_time(const char *value, time_t *result) +parse_time(const char *value, time_t *result, bool utc_default) { size_t len; int fields_num, @@ -874,7 +875,7 @@ parse_time(const char *value, time_t *result) *result = mktime(&tm); /* adjust time zone */ - if (tz_set) + if (tz_set || utc_default) { time_t ltime = time(NULL); struct tm *ptm = gmtime(<ime); @@ -983,6 +984,32 @@ parse_int(const char *value, int *result, int flags, const char **hintmsg) return true; } +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + static char * longopts_to_optstring(const struct option opts[], const size_t len) { @@ -1053,7 +1080,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) size_t len; len = option_length(options); - longopts = pgut_newarray(struct option, len + 1); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); option_copy(longopts, options, len); optstring = longopts_to_optstring(longopts, len); @@ -1095,20 +1122,22 @@ key_equals(const char *lhs, const char *rhs) /* * Get configuration from configuration file. + * Return number of parsed options */ -void +int pgut_readopt(const char *path, pgut_option options[], int elevel) { FILE *fp; char buf[1024]; char key[1024]; char value[1024]; + int parsed_options = 0; if (!options) - return; + return parsed_options; if ((fp = pgut_fopen(path, "rt", true)) == NULL) - return; + return parsed_options; while (fgets(buf, lengthof(buf), fp)) { @@ -1127,18 +1156,23 @@ pgut_readopt(const char *path, pgut_option options[], int elevel) { if (opt->allowed < SOURCE_FILE && opt->allowed != SOURCE_FILE_STRICT) - elog(elevel, "option %s cannot specified in file", opt->lname); + elog(elevel, "option %s cannot be specified in file", opt->lname); else if (opt->source <= SOURCE_FILE) + { assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } break; } } if (!options[i].type) - elog(elevel, "invalid option \"%s\"", key); + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); } } fclose(fp); + + return parsed_options; } static const char * @@ -1225,7 +1259,7 @@ get_next_token(const char *src, char *dst, const char *line) } else { - i = j = strcspn(s, "# \n\r\t\v"); + i = j = strcspn(s, "#\n\r\t\v"); memcpy(dst, s, j); } @@ -1646,7 +1680,7 @@ pgut_execute_parallel(PGconn* conn, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; @@ -1708,7 +1742,7 @@ pgut_execute_extended(PGconn* conn, const char *query, int nParams, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; @@ -1766,7 +1800,7 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (LOG_LEVEL_CONSOLE <= VERBOSE || LOG_LEVEL_FILE <= VERBOSE) + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) { int i; diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a9003f2f..0947fb7f 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -15,9 +15,9 @@ #include "pqexpbuffer.h" #include -#include #include +#include "access/xlogdefs.h" #include "logger.h" #if !defined(C_H) && !defined(__cplusplus) @@ -59,7 +59,7 @@ typedef enum pgut_optsrc typedef struct pgut_option { char type; - char sname; /* short name */ + uint8 sname; /* short name */ const char *lname; /* long name */ void *var; /* pointer to variable */ pgut_optsrc allowed; /* allowed source */ @@ -94,13 +94,6 @@ extern const char *PROGRAM_VERSION; extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; -/* ID of the main thread */ -#ifdef WIN32 -extern DWORD main_tid; -#else -extern pthread_t main_tid; -#endif - extern void pgut_help(bool details); /* @@ -118,7 +111,7 @@ extern bool in_cleanup; extern bool in_password; /* User prompts password */ extern int pgut_getopt(int argc, char **argv, pgut_option options[]); -extern void pgut_readopt(const char *path, pgut_option options[], int elevel); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel); extern void pgut_getopt_env(pgut_option options[]); extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); @@ -212,9 +205,10 @@ extern bool parse_int32(const char *value, int32 *result, int flags); extern bool parse_uint32(const char *value, uint32 *result, int flags); extern bool parse_int64(const char *value, int64 *result, int flags); extern bool parse_uint64(const char *value, uint64 *result, int flags); -extern bool parse_time(const char *value, time_t *result); +extern bool parse_time(const char *value, time_t *result, bool utc_default); extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); extern void convert_from_base_unit(int64 base_value, int base_unit, int64 *value, const char **unit); diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 00000000..82c23764 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 +#include + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static long mutex_initlock = 0; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 00000000..06460533 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "postgres_fe.h" +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +extern int pthread_lock(pthread_mutex_t *mp); + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c index afc7f07d..404965c8 100644 --- a/src/validate.c +++ b/src/validate.c @@ -11,25 +11,26 @@ #include "pg_probackup.h" #include -#include #include -static void pgBackupValidateFiles(void *arg); +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; typedef struct { - parray *files; - bool corrupted; + parray *files; + bool corrupted; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. */ int ret; -} validate_files_args; +} validate_files_arg; /* * Validate backup files. @@ -42,8 +43,9 @@ pgBackupValidate(pgBackup *backup) parray *files; bool corrupted = false; bool validation_isok = true; - pthread_t validate_threads[num_threads]; - validate_files_args *validate_threads_args[num_threads]; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; int i; /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ @@ -77,36 +79,44 @@ pgBackupValidate(pgBackup *backup) for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - __sync_lock_release(&file->lock); + pg_atomic_clear_flag(&file->lock); } + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + /* Validate files */ for (i = 0; i < num_threads; i++) { - validate_files_args *arg = pg_malloc(sizeof(validate_files_args)); + validate_files_arg *arg = &(threads_args[i]); + arg->files = files; arg->corrupted = false; /* By default there are some error */ - arg->ret = 1; + threads_args[i].ret = 1; - validate_threads_args[i] = arg; - pthread_create(&validate_threads[i], NULL, - (void *(*)(void *)) pgBackupValidateFiles, arg); + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); } /* Wait theads */ for (i = 0; i < num_threads; i++) { - pthread_join(validate_threads[i], NULL); - if (validate_threads_args[i]->corrupted) + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) corrupted = true; - if (validate_threads_args[i]->ret == 1) + if (arg->ret == 1) validation_isok = false; - pg_free(validate_threads_args[i]); } if (!validation_isok) elog(ERROR, "Data files validation failed"); + pfree(threads); + pfree(threads_args); + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); @@ -127,19 +137,19 @@ pgBackupValidate(pgBackup *backup) * rather throw a WARNING and set arguments->corrupted = true. * This is necessary to update backup status. */ -static void +static void * pgBackupValidateFiles(void *arg) { - int i; - validate_files_args *arguments = (validate_files_args *)arg; + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; pg_crc32 crc; for (i = 0; i < parray_num(arguments->files); i++) { struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); - pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (__sync_lock_test_and_set(&file->lock, 1) != 0) + if (!pg_atomic_test_set_flag(&file->lock)) continue; if (interrupted) @@ -179,14 +189,13 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : %lu. Expected %lu", - file->path, (unsigned long) file->write_size, - (unsigned long) st.st_size); + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); arguments->corrupted = true; break; } - crc = pgFileGetCRC(file); + crc = pgFileGetCRC(file->path); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", @@ -198,6 +207,8 @@ pgBackupValidateFiles(void *arg) /* Data files validation is successful */ arguments->ret = 0; + + return NULL; } /* @@ -267,7 +278,7 @@ do_validate_all(void) static void do_validate_instance(void) { - char *current_backup_id; + char *current_backup_id; int i; parray *backups; pgBackup *current_backup = NULL; @@ -279,55 +290,45 @@ do_validate_instance(void) /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backups == NULL) - elog(ERROR, "Failed to get backup list."); - /* Valiate each backup along with its xlog files. */ + /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) { - pgBackup *base_full_backup = NULL; - current_backup = (pgBackup *) parray_get(backups, i); - if (current_backup->backup_mode != BACKUP_MODE_FULL) - { - int j; - - for (j = i + 1; j < parray_num(backups); j++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (backup->backup_mode == BACKUP_MODE_FULL) - { - base_full_backup = backup; - break; - } - } - } - else - base_full_backup = current_backup; - + /* Valiate each backup along with its xlog files. */ pgBackupValidate(current_backup); - /* There is no point in wal validation for corrupted backup */ + /* Ensure that the backup has valid list of parent backups */ if (current_backup->status == BACKUP_STATUS_OK) { - if (base_full_backup == NULL) - elog(ERROR, "Valid full backup for backup %s is not found.", - base36enc(current_backup->start_time)); + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + /* Validate corresponding WAL files */ validate_wal(current_backup, arclog_path, 0, - 0, base_full_backup->tli); + 0, 0, base_full_backup->tli); } + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ if (current_backup->status == BACKUP_STATUS_CORRUPT) { int j; + corrupted_backup_found = true; current_backup_id = base36enc_dup(current_backup->start_time); for (j = i - 1; j >= 0; j--) { pgBackup *backup = (pgBackup *) parray_get(backups, j); + if (backup->backup_mode == BACKUP_MODE_FULL) break; if (backup->status != BACKUP_STATUS_OK) diff --git a/tests/__init__.py b/tests/__init__.py index d5f8b248..aeeabf2a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -60,6 +60,10 @@ def load_tests(loader, tests, pattern): # ptrack backup on replica should work correctly # archive: # immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored # 10vanilla_1.3ptrack + # 10vanilla+ # 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py index db08b45c..8b8eb71a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -29,7 +29,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -45,11 +45,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node) - node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() # Recreate backup calagoue self.init_pb(backup_dir) @@ -65,11 +61,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=["--recovery-target-action=promote"]) - node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -97,7 +89,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FIRST TIMELINE node.safe_psql( @@ -117,11 +109,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() + if self.verbose: print(node.safe_psql( "postgres", @@ -152,11 +141,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() if self.verbose: print( @@ -184,11 +169,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() + if self.verbose: print('Fourth timeline') print(node.safe_psql( @@ -200,10 +182,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() + if self.verbose: print('Fifth timeline') print(node.safe_psql( @@ -215,10 +195,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node( backup_dir, 'node', node, options=['--immediate', '--recovery-target-action=promote']) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - sleep(1) + node.slow_start() + if self.verbose: print('Sixth timeline') print(node.safe_psql( @@ -269,7 +247,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): node.append_conf( 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( archive_script_path)) - node.start() + node.slow_start() try: self.backup_node( backup_dir, 'node', node, @@ -330,7 +308,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -390,7 +368,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): f.flush() f.close() - node.start() + node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " @@ -426,7 +404,11 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): - """make node without archiving, take stream backup and turn it into replica, set replica with archiving, make archive backup from replica""" + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -441,7 +423,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -459,15 +441,18 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica, synchronous=True) + self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, take FULL backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " @@ -496,12 +481,14 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Change data on master, make PAGE backup from replica, restore taken backup and check that restored data equal to original data + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data master.psql( "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " @@ -526,7 +513,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -560,7 +547,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -586,7 +573,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -641,7 +628,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - master.start() + master.slow_start() master.psql( "postgres", @@ -668,7 +655,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA # self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -763,7 +750,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() + node.slow_start() + self.assertEqual( result, node.safe_psql( @@ -795,7 +783,7 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL 10 for this test') else: - pg_receivexlog_path = node.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') pg_receivexlog = self.run_binary( [ @@ -834,7 +822,8 @@ class ArchiveTest(ProbackupTest, unittest.TestCase): # Check data correctness node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() + node.slow_start() + self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), 'data after restore not equal to original data') diff --git a/tests/backup_test.py b/tests/backup_test.py index 5af59684..1fa74643 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -29,15 +29,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - # full backup mode - # with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - # backup_log.write(self.backup_node(node, options=["--verbose"])) - backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "FULL") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied excluded = True @@ -61,29 +57,29 @@ class BackupTest(ProbackupTest, unittest.TestCase): # print self.show_pb(node) show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PAGE") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") # Check parent backup self.assertEqual( backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # ptrack backup mode self.backup_node(backup_dir, 'node', node, backup_type="ptrack") show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['Status'], "OK") - self.assertEqual(show_backup['Mode'], "PTRACK") + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") # Check parent backup self.assertEqual( page_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['ID'])["parent-backup-id"]) + backup_id=show_backup['id'])["parent-backup-id"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -106,7 +102,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") node.stop() # Clean after yourself @@ -162,7 +158,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") # Clean after yourself @@ -227,7 +223,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.assertEqual( self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR") + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") # Clean after yourself self.del_test_dir(module_name, fname) @@ -250,12 +246,12 @@ class BackupTest(ProbackupTest, unittest.TestCase): self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -282,11 +278,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -342,7 +338,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -415,7 +411,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 2004cb14..73553a30 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -4,7 +4,7 @@ restore pg_probackup restore -B backupdir --instance instance_name [-D datadir] - [ -i backup_id | [{--time=time | --xid=xid } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] [-j num_threads] [--progress] [-q] [-v] """ diff --git a/tests/compression.py b/tests/compression.py index eff42963..aa275382 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -83,7 +83,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -98,7 +99,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -113,7 +115,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -187,7 +190,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -202,7 +206,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -217,7 +222,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -294,7 +300,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -309,7 +316,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -324,7 +332,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -401,7 +410,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -416,7 +426,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -431,7 +442,8 @@ class CompressionTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() diff --git a/tests/delete_test.py b/tests/delete_test.py index 039f3606..4afb15ae 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -44,13 +44,13 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.backup_node(backup_dir, 'node', node) show_backups = self.show_pb(backup_dir, 'node') - id_1 = show_backups[0]['ID'] - id_2 = show_backups[1]['ID'] - id_3 = show_backups[2]['ID'] + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] self.delete_pb(backup_dir, 'node', id_2) show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(show_backups[0]['ID'], id_1) - self.assertEqual(show_backups[1]['ID'], id_3) + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) # Clean after yourself self.del_test_dir(module_name, fname) @@ -82,15 +82,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) @@ -122,15 +122,15 @@ class DeleteTest(ProbackupTest, unittest.TestCase): self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], "FULL") - self.assertEqual(show_backups[0]['Status'], "OK") - self.assertEqual(show_backups[1]['Mode'], "FULL") - self.assertEqual(show_backups[1]['Status'], "OK") + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index 303c7c58..40450016 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -508,10 +508,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") @@ -946,11 +943,8 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1191,7 +1185,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): f.close self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK', + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', "Backup Status should be OK") # Clean after yourself @@ -1264,7 +1258,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): repr(e.message), self.cmd)) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['Status'] == 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', "Backup Status should be ERROR") # Clean after yourself diff --git a/tests/exclude.py b/tests/exclude.py index b85f64a7..48b7889c 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -43,25 +43,33 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): "WHERE oid = pg_my_temp_schema()")[0][0] conn.commit() - temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace("pg_", "") + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") conn.commit() conn.execute("create index test_idx on test (generate_series)") conn.commit() - heap_path = conn.execute("select pg_relation_filepath('test')")[0][0] + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] conn.commit() - index_path = conn.execute("select pg_relation_filepath('test_idx')")[0][0] + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] conn.commit() heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] conn.commit() - toast_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] conn.commit() - toast_idx_path = conn.execute("select pg_relation_filepath('{0}.{1}')".format(temp_toast_schema_name, "pg_toast_" + str(heap_oid) + "_index"))[0][0] + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] conn.commit() temp_table_filename = os.path.basename(heap_path) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 4080f4b8..35f58406 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -25,6 +25,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--replica-timeout=timeout] pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-C] [--stream [-S slot-name]] [--backup-pg-log] @@ -49,26 +50,31 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] - pg_probackup restore -B backup-dir --instance=instance_name [-D pgdata-dir] [-i backup-id] [--progress] - [--time=time|--xid=xid [--inclusive=boolean]] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--timeline=timeline] [-T OLDDIR=NEWDIR] [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] + [--no-validate] pg_probackup validate -B backup-dir [--instance=instance_name] [-i backup-id] [--progress] - [--time=time|--xid=xid [--inclusive=boolean]] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-name=target-name] [--timeline=timeline] pg_probackup show -B backup-dir [--instance=instance_name [-i backup-id]] + [--format=format] pg_probackup delete -B backup-dir --instance=instance_name [--wal] [-i backup-id | --expired] + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + pg_probackup add-instance -B backup-dir -D pgdata-dir --instance=instance_name diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index a1257673..35e212c3 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.17 \ No newline at end of file +pg_probackup 2.0.18 \ No newline at end of file diff --git a/tests/false_positive.py b/tests/false_positive.py index 909d69fc..1884159b 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -191,7 +191,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, @@ -290,7 +290,7 @@ class FalsePositive(ProbackupTest, unittest.TestCase): node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() + node.slow_start() # Logical comparison self.assertEqual( result, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a776599c..0d04d898 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -12,6 +12,7 @@ import select import psycopg2 from time import sleep import re +import json idx_ptrack = { 't_heap': { @@ -111,6 +112,39 @@ class ProbackupException(Exception): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +def slow_start(self, replica=False): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue + + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() @@ -204,6 +238,8 @@ class ProbackupTest(object): os.makedirs(real_base_dir) node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) node.should_rm_dirs = True node.init( initdb_params=initdb_params, allow_streaming=set_replication) @@ -578,6 +614,16 @@ class ProbackupTest(object): return self.run_pb(cmd_list + options, async, gdb) + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + def restore_node( self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[] @@ -598,7 +644,7 @@ class ProbackupTest(object): def show_pb( self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False + options=[], as_text=False, as_json=True ): backup_list = [] @@ -613,63 +659,83 @@ class ProbackupTest(object): if backup_id: cmd_list += ["-i", backup_id] + if as_json: + cmd_list += ["--format=json"] + if as_text: # You should print it when calling as_text=true return self.run_pb(cmd_list + options) # get show result as list of lines - show_splitted = self.run_pb(cmd_list + options).splitlines() - if instance is not None and backup_id is None: - # cut header(ID, Mode, etc) from show as single string - header = show_splitted[1:2][0] - # cut backup records from show as single list - # with string for every backup record - body = show_splitted[3:] - # inverse list so oldest record come first - body = body[::-1] - # split string in list with string for every header element - header_split = re.split(" +", header) - # Remove empty items - for i in header_split: - if i == '': - header_split.remove(i) + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: continue - header_split = [ - header_element.rstrip() for header_element in header_split - ] - for backup_record in body: - backup_record = backup_record.rstrip() - # split list with str for every backup record element - backup_record_split = re.split(" +", backup_record) - # Remove empty items - for i in backup_record_split: - if i == '': - backup_record_split.remove(i) - if len(header_split) != len(backup_record_split): - print(warning.format( - header=header, body=body, - header_split=header_split, - body_split=backup_record_split) - ) - exit(1) - new_dict = dict(zip(header_split, backup_record_split)) - backup_list.append(new_dict) + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) return backup_list else: - # cut out empty lines and lines started with # - # and other garbage then reconstruct it as dictionary - # print show_splitted - sanitized_show = [item for item in show_splitted if item] - sanitized_show = [ - item for item in sanitized_show if not item.startswith('#') - ] - # print sanitized_show - for line in sanitized_show: - name, var = line.partition(" = ")[::2] - var = var.strip('"') - var = var.strip("'") - specific_record[name.strip()] = var - return specific_record + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record def validate_pb( self, backup_dir, instance=None, @@ -851,13 +917,26 @@ class ProbackupTest(object): return num def switch_wal_segment(self, node): - """ Execute pg_switch_wal/xlog() in given node""" - if self.version_to_num( - node.safe_psql("postgres", "show server_version") - ) >= self.version_to_num('10.0'): - node.safe_psql("postgres", "select pg_switch_wal()") + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") else: - node.safe_psql("postgres", "select pg_switch_xlog()") + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") sleep(1) def get_version(self, node): diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 00000000..1be3dd8b --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py index a66d346e..8bd473fa 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -12,6 +12,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_help_1(self): """help options""" + self.maxDiff = None fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: @@ -199,7 +200,8 @@ class OptionTest(ProbackupTest, unittest.TestCase): self.add_instance(backup_dir, 'node', node) # invalid option in pg_probackup.conf - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: conf.write("TIMELINEID=1\n") try: @@ -209,7 +211,7 @@ class OptionTest(ProbackupTest, unittest.TestCase): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: invalid option "TIMELINEID"\n', + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself diff --git a/tests/page.py b/tests/page.py index 9a7260c2..ef7122b6 100644 --- a/tests/page.py +++ b/tests/page.py @@ -3,8 +3,6 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess -import time - module_name = 'page' @@ -33,8 +31,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir="{0}/{1}/node_restored".format(module_name, fname)) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -49,32 +46,27 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "create table t_heap tablespace somedata as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") + node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose'] - ) + options=['--log-level-file=verbose']) self.backup_node( - backup_dir, 'node', node, backup_type='page' - ) + backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -87,8 +79,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"] - ) + "--recovery-target-action=promote"]) # Physical comparison if self.paranoia: @@ -97,21 +88,17 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() - - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node_restored.slow_start() # Logical comparison result1 = node.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + result2 = node_restored.safe_psql( "postgres", - "select * from t_heap" - ) + "select * from t_heap") + self.assertEqual(result1, result2) # Clean after yourself @@ -175,7 +162,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=full_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -188,7 +175,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): backup_id=page_backup_id, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -254,7 +241,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -271,7 +259,8 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -349,10 +338,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from pgbench_accounts") @@ -526,3 +512,130 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index a67f3dd4..bd40f16d 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -63,7 +63,7 @@ class ArchiveCheck(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] self.assertEqual( 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') diff --git a/tests/ptrack.py b/tests/ptrack.py index 9c21ff55..c2d6abff 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -268,7 +268,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -430,7 +431,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node_restored.append_conf( @@ -503,7 +505,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # Physical comparison if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) node.start() @@ -583,10 +586,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -604,13 +604,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -687,10 +685,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) ) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -711,13 +707,11 @@ class PtrackTest(ProbackupTest, unittest.TestCase): ) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) @@ -811,7 +805,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_page_pgpro417(self): - """Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail""" + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -880,7 +877,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_full_pgpro417(self): - """Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail""" + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -954,7 +954,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_create_db(self): - """Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense""" + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1017,7 +1020,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1046,7 +1050,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1151,17 +1156,15 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET RESTORED PGDATA AND COMPARE if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() - while node_restored.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) result_new = node_restored.safe_psql( "postgres", "select * from t_heap") @@ -1229,7 +1232,8 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE @@ -1240,7 +1244,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_drop_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1321,7 +1328,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1379,16 +1389,14 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() # COMPARE LOGICAL CONTENT result_new = restored_node.safe_psql( @@ -1416,17 +1424,14 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", "select * from t_heap") @@ -1437,7 +1442,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_ptrack_multiple_segments(self): - """Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup""" + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1446,9 +1454,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'ptrack_enable': 'on', 'fsync': 'off', 'shared_buffers': '128MB', - 'maintenance_work_mem': '1GB', 'autovacuum': 'off', - 'full_page_writes': 'off'} + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } ) self.init_pb(backup_dir) @@ -1514,17 +1523,14 @@ class PtrackTest(ProbackupTest, unittest.TestCase): # GET PHYSICAL CONTENT FROM NODE_RESTORED if self.paranoia: - pgdata_restored = self.pgdata_content(restored_node.data_dir, ignore_ptrack=False) + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE restored_node.append_conf( "postgresql.auto.conf", "port = {0}".format(restored_node.port)) - restored_node.start() - while restored_node.safe_psql( - "postgres", - "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + restored_node.slow_start() result_new = restored_node.safe_psql( "postgres", diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index cf8702b8..784751ef 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -46,7 +46,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_btree') @@ -103,7 +103,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_gist') @@ -172,7 +172,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') @@ -242,7 +242,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py index 51e90503..928608c4 100644 --- a/tests/ptrack_truncate.py +++ b/tests/ptrack_truncate.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -100,7 +100,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index ad2f65bc..0409cae3 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -45,7 +45,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index dc1e5fd8..f0cd3bbd 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -43,7 +43,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') @@ -111,7 +111,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'vacuum freeze t_heap') diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index a49d6075..ec12c9e2 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'vacuum full t_heap') diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 5fd85056..5c84c7e8 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -44,7 +44,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id > 128;') node.safe_psql('postgres', 'vacuum t_heap') @@ -116,7 +116,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Take PTRACK backup to clean every ptrack - self.backup_node(backup_dir, 'replica', replica, options=['-j100', + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) master.safe_psql('postgres', 'delete from t_heap where id > 128;') diff --git a/tests/replica.py b/tests/replica.py index 5d6ecfb1..d74c375c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -50,7 +50,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_replica(master, replica) # Check data correctness on replica - replica.start(["-t", "600"]) + replica.slow_start(replica=True) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -82,7 +82,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -113,7 +113,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -143,7 +143,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -166,7 +166,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): # Settings for Replica self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start(["-t", "600"]) + replica.slow_start(replica=True) # Check data correctness on replica after = replica.safe_psql("postgres", "SELECT * FROM t_heap") @@ -200,7 +200,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -231,7 +231,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - node.start() + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -260,7 +260,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'master', master) # force more frequent wal switch master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() + master.slow_start() replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -287,15 +287,7 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.start(["-t", "600"]) - - time.sleep(1) - self.assertEqual( - master.safe_psql( - "postgres", - "select exists(select * from pg_stat_replication)" - ).rstrip(), - 't') + replica.start() # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py index 4567f37b..c33a1e29 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -53,10 +53,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): recovery_conf = os.path.join(node.data_dir, "recovery.conf") self.assertEqual(os.path.isfile(recovery_conf), True) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -104,10 +101,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -149,11 +143,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start(params=['-t', '10']) - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '10', '-c', '2', '--no-vacuum']) @@ -181,11 +171,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -234,11 +220,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -297,11 +279,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -366,11 +344,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) self.assertEqual( @@ -379,6 +353,153 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" @@ -420,11 +541,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -479,11 +596,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -535,11 +648,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) @@ -602,11 +711,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -674,11 +779,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): options=["-j", "4", "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute( @@ -718,7 +819,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): con.commit() backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # 1 - Try to restore to existing directory node.stop() @@ -769,10 +870,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test") self.assertEqual(result[0][0], 1) @@ -785,8 +884,8 @@ class RestoreTest(ProbackupTest, unittest.TestCase): backup_dir, 'node', node, backup_type="page") show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['Status'], "OK") - self.assertEqual(show_pb[2]['Status'], "OK") + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") node.stop() node.cleanup() @@ -802,10 +901,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.execute("postgres", "SELECT id FROM test OFFSET 1") self.assertEqual(result[0][0], 2) @@ -829,7 +925,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Full backup self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -845,8 +941,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # First page backup self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: @@ -862,8 +959,9 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # Second page backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") node.stop() node.cleanup() @@ -879,10 +977,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) @@ -933,10 +1028,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -985,11 +1077,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) - + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1037,10 +1125,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) @@ -1095,10 +1180,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) @@ -1147,10 +1229,7 @@ class RestoreTest(ProbackupTest, unittest.TestCase): "--recovery-target-name=savepoint", "--recovery-target-action=promote"]) - node.start() - while node.safe_psql( - "postgres", "select pg_is_in_recovery()") == 't\n': - time.sleep(1) + node.slow_start() result_new = node.safe_psql("postgres", "select * from t_heap") res = node.psql("postgres", "select * from t_heap_1") diff --git a/tests/retention_test.py b/tests/retention_test.py index 2d4cac37..652f7c39 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -14,7 +14,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -24,7 +25,9 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.set_archiving(backup_dir, 'node', node) node.start() - with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") # Make backups to be purged @@ -57,7 +60,7 @@ class RetentionTest(ProbackupTest, unittest.TestCase): for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - #wal_name_b = wal_name.encode('ascii') + # wal_name_b = wal_name.encode('ascii') self.assertEqual(wal_name[8:] > min_wal[8:], True) self.assertEqual(wal_name[8:] > max_wal[8:], True) @@ -68,7 +71,8 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/show_test.py b/tests/show_test.py index 74bd0341..931da184 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -36,6 +36,35 @@ class OptionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" @@ -81,3 +110,94 @@ class OptionTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index 06ea1ea3..ab091c57 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -748,6 +748,93 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_validate_instance_with_corrupted_full(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, @@ -908,7 +995,8 @@ class ValidateTest(ProbackupTest, unittest.TestCase): backup_id = self.backup_node(backup_dir, 'node', node) target_xid = None with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() target_xid = res[0][0] @@ -1041,7 +1129,10 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): - """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), @@ -1083,7 +1174,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): else: walfile = node.safe_psql( 'postgres', - 'select pg_walfile_name(pg_current_wal_location())').rstrip() + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -1134,12 +1225,12 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['Status'], + self.show_pb(backup_dir, 'node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup STATUS should be "OK"') # Clean after yourself @@ -1208,7 +1299,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['Status'], + self.show_pb(backup_dir, 'node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Clean after yourself @@ -1316,7 +1407,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) node2.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) - node2.start() + node2.slow_start() timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] @@ -1405,7 +1496,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): except ProbackupException as e: pass self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') self.set_archiving(backup_dir, 'node', node) node.reload() self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1440,14 +1531,19 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') os.rename(file_new, file) try: @@ -1459,14 +1555,15 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['Status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') # Clean after yourself self.del_test_dir(module_name, fname) @@ -1537,13 +1634,13 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') os.rename(file_new, file) file = os.path.join( @@ -1562,13 +1659,72 @@ class ValidateTest(ProbackupTest, unittest.TestCase): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['Status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['Status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['Status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 00000000..14864181 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 00000000..c869e485 --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 00000000..a4f75553 --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + +