v2.11: C Implementation of Archive Get

Bug Fixes:

* Fix possible truncated WAL segments when an error occurs mid-write. (Reported by blogh.)
* Fix info command missing WAL min/max when stanza specified. (Fixed by Stefan Fercot.)
* Fix non-compliant JSON for options passed from C to Perl. (Reported by Leo Khomenko.)

Improvements:

* The archive-get command is implemented entirely in C.
* Enable socket keep-alive on older Perl versions. (Contributed by Marc Cousin.)
* Error when parameters are passed to a command that does not accept parameters. (Suggested by Jason O'Donnell.)
* Add hints when unable to find a WAL segment in the archive. (Suggested by Hans-Jürgen Schönig.)
* Improve error when hostname cannot be found in a certificate. (Suggested by James Badger.)
* Add additional options to backup.manifest for debugging purposes. (Contributed by blogh.)

README.md

@@ -4,7 +4,7 @@
 
 pgBackRest aims to be a simple, reliable backup and restore solution that can seamlessly scale up to the largest databases and workloads by utilizing algorithms that are optimized for database-specific requirements.
 
-pgBackRest [v2.10](https://github.com/pgbackrest/pgbackrest/releases/tag/release/2.10) is the current stable release. Release notes are on the [Releases](http://www.pgbackrest.org/release.html) page.
+pgBackRest [v2.11](https://github.com/pgbackrest/pgbackrest/releases/tag/release/2.11) is the current stable release. Release notes are on the [Releases](http://www.pgbackrest.org/release.html) page.
 
 Documentation for v1 can be found [here](http://www.pgbackrest.org/1). No further releases are planned for v1 because v2 is backward-compatible with v1 options and repositories.
 
@@ -90,7 +90,7 @@ pgBackRest includes support for versions down to 8.3, since older versions of Po
 
 pgBackRest strives to be easy to configure and operate:
 
-- [User guide](http://www.pgbackrest.org/user-guide.html) for Debian & Ubuntu / PostgreSQL 9.4.
+- [User guide](http://www.pgbackrest.org/user-guide.html) for Debian & Ubuntu / PostgreSQL 10.
 
 - [Command reference](http://www.pgbackrest.org/command.html) for command-line operations.
 

doc/RELEASE.md

@@ -30,7 +30,8 @@ if (!$bTest)
 
 - Copy coverage report:
 ```
-cd doc/site
+cd <pgbackrest-base>/doc/site
+rm -rf coverage
 cp -r ../../test/coverage/c coverage
 ```
 
@@ -63,7 +64,7 @@ with:
 
 - In `doc/site/coverage` replace:
 ```
-            <td class="headerValue">all C unit</td>
+            <td class="headerValue">all.lcov</td>
 ```
 with:
 ```
@@ -72,6 +73,6 @@ with:
 
 - Switch to prior dir and copy coverage:
 ```
-doc/site/prior/X.XX
+cd ../prior/X.XX
 cp -r ../../coverage .
 ```

doc/xml/release.xml

@@ -12,7 +12,7 @@
     </intro>
 
     <release-list>
-        <release date="XXXX-XX-XX" version="2.11dev" title="UNDER DEVELOPMENT">
+        <release date="2019-03-11" version="2.11" title="C Implementation of Archive Get">
             <release-core-list>
                 <release-bug-list>
                     <release-item>

lib/pgBackRest/Version.pm

@@ -39,7 +39,7 @@ push @EXPORT, qw(projectBin projectBinSet);
 # Defines the current version of the BackRest executable.  The version number is used to track features but does not affect what
 # repositories or manifests can be read - that's the job of the format number.
 #-----------------------------------------------------------------------------------------------------------------------------------
-use constant PROJECT_VERSION                                        => '2.11dev';
+use constant PROJECT_VERSION                                        => '2.11';
     push @EXPORT, qw(PROJECT_VERSION);
 
 # Repository Format Number

src/perl/embed.auto.c

@@ -20719,7 +20719,7 @@ static const EmbeddedModule embeddedModule[] =
             "\n"
             "push @EXPORT, qw(projectBin projectBinSet);\n"
             "\n\n\n\n\n\n"
-            "use constant PROJECT_VERSION => '2.11dev';\n"
+            "use constant PROJECT_VERSION => '2.11';\n"
             "push @EXPORT, qw(PROJECT_VERSION);\n"
             "\n\n\n\n\n\n"
             "use constant REPOSITORY_FORMAT => 5;\n"

src/version.h

@@ -23,6 +23,6 @@ repository will be invalid unless migration functions are written.
 /***********************************************************************************************************************************
 Software version.  Currently this value is maintained in Version.pm and updated by test.pl.
 ***********************************************************************************************************************************/
-#define PROJECT_VERSION                                             "2.11dev"
+#define PROJECT_VERSION                                             "2.11"
 
 #endif

test/code-count/file-type.yaml

@@ -159,18 +159,10 @@ lib/pgBackRest/Archive/Common.pm:
   class: core
   type: perl
 
-lib/pgBackRest/Archive/Get/Async.pm:
-  class: core
-  type: perl
-
 lib/pgBackRest/Archive/Get/File.pm:
   class: core
   type: perl
 
-lib/pgBackRest/Archive/Get/Get.pm:
-  class: core
-  type: perl
-
 lib/pgBackRest/Archive/Info.pm:
   class: core
   type: perl
@@ -291,10 +283,6 @@ lib/pgBackRest/Expire.pm:
   class: core
   type: perl
 
-lib/pgBackRest/Info.pm:
-  class: core
-  type: perl
-
 lib/pgBackRest/InfoCommon.pm:
   class: core
   type: perl
@@ -543,6 +531,14 @@ src/command/archive/get/get.h:
   class: core
   type: c/h
 
+src/command/archive/get/protocol.c:
+  class: core
+  type: c
+
+src/command/archive/get/protocol.h:
+  class: core
+  type: c/h
+
 src/command/archive/push/push.c:
   class: core
   type: c
@@ -583,10 +579,74 @@ src/command/info/info.h:
   class: core
   type: c/h
 
+src/command/local/local.c:
+  class: core
+  type: c
+
+src/command/local/local.h:
+  class: core
+  type: c/h
+
+src/command/remote/remote.c:
+  class: core
+  type: c
+
+src/command/remote/remote.h:
+  class: core
+  type: c/h
+
 src/common/assert.h:
   class: core
   type: c/h
 
+src/common/compress/gzip/common.c:
+  class: core
+  type: c
+
+src/common/compress/gzip/common.h:
+  class: core
+  type: c/h
+
+src/common/compress/gzip/compress.c:
+  class: core
+  type: c
+
+src/common/compress/gzip/compress.h:
+  class: core
+  type: c/h
+
+src/common/compress/gzip/decompress.c:
+  class: core
+  type: c
+
+src/common/compress/gzip/decompress.h:
+  class: core
+  type: c/h
+
+src/common/crypto/cipherBlock.c:
+  class: core
+  type: c
+
+src/common/crypto/cipherBlock.h:
+  class: core
+  type: c/h
+
+src/common/crypto/common.c:
+  class: core
+  type: c
+
+src/common/crypto/common.h:
+  class: core
+  type: c/h
+
+src/common/crypto/hash.c:
+  class: core
+  type: c
+
+src/common/crypto/hash.h:
+  class: core
+  type: c/h
+
 src/common/debug.c:
   class: core
   type: c
@@ -627,6 +687,14 @@ src/common/error.h:
   class: core
   type: c/h
 
+src/common/exec.c:
+  class: core
+  type: c
+
+src/common/exec.h:
+  class: core
+  type: c/h
+
 src/common/exit.c:
   class: core
   type: c
@@ -703,6 +771,14 @@ src/common/io/filter/size.h:
   class: core
   type: c/h
 
+src/common/io/handleRead.c:
+  class: core
+  type: c
+
+src/common/io/handleRead.h:
+  class: core
+  type: c/h
+
 src/common/io/handleWrite.c:
   class: core
   type: c
@@ -923,30 +999,6 @@ src/common/wait.h:
   class: core
   type: c/h
 
-src/compress/gzip.c:
-  class: core
-  type: c
-
-src/compress/gzip.h:
-  class: core
-  type: c/h
-
-src/compress/gzipCompress.c:
-  class: core
-  type: c
-
-src/compress/gzipCompress.h:
-  class: core
-  type: c/h
-
-src/compress/gzipDecompress.c:
-  class: core
-  type: c
-
-src/compress/gzipDecompress.h:
-  class: core
-  type: c/h
-
 src/config/config.auto.c:
   class: core/auto
   type: c
@@ -979,6 +1031,14 @@ src/config/define.h:
   class: core
   type: c/h
 
+src/config/exec.c:
+  class: core
+  type: c
+
+src/config/exec.h:
+  class: core
+  type: c/h
+
 src/config/load.c:
   class: core
   type: c
@@ -999,27 +1059,11 @@ src/config/parse.h:
   class: core
   type: c/h
 
-src/crypto/cipherBlock.c:
+src/config/protocol.c:
   class: core
   type: c
 
-src/crypto/cipherBlock.h:
+src/config/protocol.h:
-  class: core
-  type: c/h
-
-src/crypto/crypto.c:
-  class: core
-  type: c
-
-src/crypto/crypto.h:
-  class: core
-  type: c/h
-
-src/crypto/hash.c:
-  class: core
-  type: c
-
-src/crypto/hash.h:
   class: core
   type: c/h
 
@@ -1243,6 +1287,54 @@ src/postgres/version.h:
   class: core
   type: c/h
 
+src/protocol/client.c:
+  class: core
+  type: c
+
+src/protocol/client.h:
+  class: core
+  type: c/h
+
+src/protocol/command.c:
+  class: core
+  type: c
+
+src/protocol/command.h:
+  class: core
+  type: c/h
+
+src/protocol/helper.c:
+  class: core
+  type: c
+
+src/protocol/helper.h:
+  class: core
+  type: c/h
+
+src/protocol/parallel.c:
+  class: core
+  type: c
+
+src/protocol/parallel.h:
+  class: core
+  type: c/h
+
+src/protocol/parallelJob.c:
+  class: core
+  type: c
+
+src/protocol/parallelJob.h:
+  class: core
+  type: c/h
+
+src/protocol/server.c:
+  class: core
+  type: c
+
+src/protocol/server.h:
+  class: core
+  type: c/h
+
 src/storage/driver/posix/common.c:
   class: core
   type: c
@@ -1275,6 +1367,30 @@ src/storage/driver/posix/storage.h:
   class: core
   type: c/h
 
+src/storage/driver/remote/fileRead.c:
+  class: core
+  type: c
+
+src/storage/driver/remote/fileRead.h:
+  class: core
+  type: c/h
+
+src/storage/driver/remote/protocol.c:
+  class: core
+  type: c
+
+src/storage/driver/remote/protocol.h:
+  class: core
+  type: c/h
+
+src/storage/driver/remote/storage.c:
+  class: core
+  type: c
+
+src/storage/driver/remote/storage.h:
+  class: core
+  type: c/h
+
 src/storage/driver/s3/fileRead.c:
   class: core
   type: c
@@ -1451,31 +1567,23 @@ test/lib/pgBackRestTest/Env/S3EnvTest.pm:
   class: test/harness
   type: perl
 
-test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm:
-  class: test/module
-  type: perl
-
-test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm:
-  class: test/module
-  type: perl
-
-test/lib/pgBackRestTest/Module/Archive/ArchiveInfoUnitPerlTest.pm:
-  class: test/module
-  type: perl
-
-test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm:
-  class: test/module
-  type: perl
-
 test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm:
   class: test/module
   type: perl
 
-test/lib/pgBackRestTest/Module/Backup/BackupInfoUnitPerlTest.pm:
+test/lib/pgBackRestTest/Module/Backup/BackupUnitPerlTest.pm:
   class: test/module
   type: perl
 
-test/lib/pgBackRestTest/Module/Backup/BackupUnitPerlTest.pm:
+test/lib/pgBackRestTest/Module/Command/CommandArchiveCommonPerlTest.pm:
+  class: test/module
+  type: perl
+
+test/lib/pgBackRestTest/Module/Command/CommandArchiveGetPerlTest.pm:
+  class: test/module
+  type: perl
+
+test/lib/pgBackRestTest/Module/Command/CommandArchivePushPerlTest.pm:
   class: test/module
   type: perl
 
@@ -1507,7 +1615,11 @@ test/lib/pgBackRestTest/Module/Common/CommonLogPerlTest.pm:
   class: test/module
   type: perl
 
-test/lib/pgBackRestTest/Module/Info/InfoUnitPerlTest.pm:
+test/lib/pgBackRestTest/Module/Info/InfoInfoArchivePerlTest.pm:
+  class: test/module
+  type: perl
+
+test/lib/pgBackRestTest/Module/Info/InfoInfoBackupPerlTest.pm:
   class: test/module
   type: perl
 
@@ -1639,15 +1751,15 @@ test/src/common/harnessTls.h:
   class: test/harness
   type: c/h
 
-test/src/module/archive/commonTest.c:
+test/src/module/command/archiveCommonTest.c:
   class: test/module
   type: c
 
-test/src/module/archive/getTest.c:
+test/src/module/command/archiveGetTest.c:
   class: test/module
   type: c
 
-test/src/module/archive/pushTest.c:
+test/src/module/command/archivePushTest.c:
   class: test/module
   type: c
 
@@ -1659,10 +1771,22 @@ test/src/module/command/controlTest.c:
   class: test/module
   type: c
 
+test/src/module/command/helpTest.c:
+  class: test/module
+  type: c
+
 test/src/module/command/infoTest.c:
   class: test/module
   type: c
 
+test/src/module/command/localTest.c:
+  class: test/module
+  type: c
+
+test/src/module/command/remoteTest.c:
+  class: test/module
+  type: c
+
 test/src/module/common/assertOffTest.c:
   class: test/module
   type: c
@@ -1671,6 +1795,14 @@ test/src/module/common/assertOnTest.c:
   class: test/module
   type: c
 
+test/src/module/common/compressGzipTest.c:
+  class: test/module
+  type: c
+
+test/src/module/common/cryptoTest.c:
+  class: test/module
+  type: c
+
 test/src/module/common/debugOffTest.c:
   class: test/module
   type: c
@@ -1687,6 +1819,10 @@ test/src/module/common/errorTest.c:
   class: test/module
   type: c
 
+test/src/module/common/execTest.c:
+  class: test/module
+  type: c
+
 test/src/module/common/exitTest.c:
   class: test/module
   type: c
@@ -1775,10 +1911,6 @@ test/src/module/common/waitTest.c:
   class: test/module
   type: c
 
-test/src/module/compress/gzipTest.c:
-  class: test/module
-  type: c
-
 test/src/module/config/configTest.c:
   class: test/module
   type: c
@@ -1787,6 +1919,10 @@ test/src/module/config/defineTest.c:
   class: test/module
   type: c
 
+test/src/module/config/execTest.c:
+  class: test/module
+  type: c
+
 test/src/module/config/loadTest.c:
   class: test/module
   type: c
@@ -1795,19 +1931,7 @@ test/src/module/config/parseTest.c:
   class: test/module
   type: c
 
-test/src/module/crypto/cipherBlockTest.c:
+test/src/module/config/protocolTest.c:
-  class: test/module
-  type: c
-
-test/src/module/crypto/cryptoTest.c:
-  class: test/module
-  type: c
-
-test/src/module/crypto/hashTest.c:
-  class: test/module
-  type: c
-
-test/src/module/help/helpTest.c:
   class: test/module
   type: c
 
@@ -1843,10 +1967,18 @@ test/src/module/postgres/pageChecksumTest.c:
   class: test/module
   type: c
 
+test/src/module/protocol/protocolTest.c:
+  class: test/module
+  type: c
+
 test/src/module/storage/posixTest.c:
   class: test/module
   type: c
 
+test/src/module/storage/remoteTest.c:
+  class: test/module
+  type: c
+
 test/src/module/storage/s3Test.c:
   class: test/module
   type: c