From 8d6a08a32b1221db5b6eae0db615731d19754bc8 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 3 Nov 2017 13:57:58 -0400 Subject: [PATCH] Library code for repository encryption support. --- doc/xml/release.xml | 6 + lib/pgBackRest/Common/Exception.pm | 4 +- lib/pgBackRest/Storage/Base.pm | 8 + lib/pgBackRest/Storage/Filter/CipherBlock.pm | 177 +++++++++++ libc/LibC.h | 82 +++++ libc/LibC.xs | 5 +- libc/Makefile.PL | 4 + libc/build/lib/pgBackRestLibC/Build.pm | 15 + libc/lib/pgBackRest/LibCAuto.pm | 11 + libc/typemap | 1 + libc/xs/cipher/block.xs | 81 +++++ libc/xs/cipher/block.xsh | 15 + libc/xs/cipher/random.xs | 19 ++ src/cipher/block.c | 255 ++++++++++++++++ src/cipher/block.h | 20 ++ src/cipher/cipher.h | 14 + src/cipher/random.c | 15 + src/cipher/random.h | 10 + src/common/error.c | 2 +- src/common/error.h | 4 +- src/common/errorType.c | 1 + src/common/errorType.h | 1 + test/Vagrantfile | 2 +- .../pgBackRestTest/Common/ContainerTest.pm | 4 +- test/lib/pgBackRestTest/Common/DefineTest.pm | 39 +++ test/lib/pgBackRestTest/Common/JobTest.pm | 2 +- .../Storage/StorageFilterCipherBlockTest.pm | 285 ++++++++++++++++++ test/src/module/cipher/blockTest.c | 180 +++++++++++ test/src/module/cipher/randomTest.c | 32 ++ 29 files changed, 1285 insertions(+), 9 deletions(-) create mode 100644 lib/pgBackRest/Storage/Filter/CipherBlock.pm create mode 100644 libc/typemap create mode 100644 libc/xs/cipher/block.xs create mode 100644 libc/xs/cipher/block.xsh create mode 100644 libc/xs/cipher/random.xs create mode 100644 src/cipher/block.c create mode 100644 src/cipher/block.h create mode 100644 src/cipher/cipher.h create mode 100644 src/cipher/random.c create mode 100644 src/cipher/random.h create mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockTest.pm create mode 100644 test/src/module/cipher/blockTest.c create mode 100644 test/src/module/cipher/randomTest.c diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 0c73402d2..4baf4e6b5 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -12,6 +12,12 @@ + + +

Repository encryption support.

+ + +

Add list type for options. The hash type was being used for lists with an additional flag (`value-hash`) to indicate that it was not really a hash.

diff --git a/lib/pgBackRest/Common/Exception.pm b/lib/pgBackRest/Common/Exception.pm index 92450e34f..0779216c2 100644 --- a/lib/pgBackRest/Common/Exception.pm +++ b/lib/pgBackRest/Common/Exception.pm @@ -157,7 +157,9 @@ use constant ERROR_PATH_EXISTS => ERROR_MIN use constant ERROR_FILE_EXISTS => ERROR_MINIMUM + 68; push @EXPORT, qw(ERROR_FILE_EXISTS); use constant ERROR_MEMORY => ERROR_MINIMUM + 69; # Thrown by C library - push @EXPORT, qw(ERROR_CRYPT); + push @EXPORT, qw(ERROR_MEMORY); +use constant ERROR_CIPHER => ERROR_MINIMUM + 70; + push @EXPORT, qw(ERROR_CIPHER); use constant ERROR_INVALID_VALUE => ERROR_MAXIMUM - 2; push @EXPORT, qw(ERROR_INVALID_VALUE); diff --git a/lib/pgBackRest/Storage/Base.pm b/lib/pgBackRest/Storage/Base.pm index a0311b152..3ae8bf76c 100644 --- a/lib/pgBackRest/Storage/Base.pm +++ b/lib/pgBackRest/Storage/Base.pm @@ -24,6 +24,14 @@ use constant STORAGE_COMPRESS => 'compress use constant STORAGE_DECOMPRESS => 'decompress'; push @EXPORT, qw(STORAGE_DECOMPRESS); +#################################################################################################################################### +# Cipher constants +#################################################################################################################################### +use constant STORAGE_ENCRYPT => 'encrypt'; + push @EXPORT, qw(STORAGE_ENCRYPT); +use constant STORAGE_DECRYPT => 'decrypt'; + push @EXPORT, qw(STORAGE_DECRYPT); + #################################################################################################################################### # Capability constants #################################################################################################################################### diff --git a/lib/pgBackRest/Storage/Filter/CipherBlock.pm b/lib/pgBackRest/Storage/Filter/CipherBlock.pm new file mode 100644 index 000000000..b828e17da --- /dev/null +++ b/lib/pgBackRest/Storage/Filter/CipherBlock.pm @@ -0,0 +1,177 @@ +#################################################################################################################################### +# Block Cipher Filter +#################################################################################################################################### +package pgBackRest::Storage::Filter::CipherBlock; +use parent 'pgBackRest::Common::Io::Filter'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Io::Base; +use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:cipher); +use pgBackRest::Storage::Base; + +#################################################################################################################################### +# Package name constant +#################################################################################################################################### +use constant STORAGE_FILTER_CIPHER_BLOCK => __PACKAGE__; + push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK); + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oParent, + $strCipherType, + $tCipherPass, + $strMode, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oParent', trace => true}, + {name => 'strCipherType', trace => true}, + {name => 'tCipherPass', trace => true}, + {name => 'strMode', optional => true, default => STORAGE_ENCRYPT, trace => true}, + ); + + # Bless with new class + my $self = $class->SUPER::new($oParent); + bless $self, $class; + + # Check mode is valid + $self->{strMode} = $strMode; + + if (!($self->{strMode} eq STORAGE_ENCRYPT || $self->{strMode} eq STORAGE_DECRYPT)) + { + confess &log(ASSERT, "unknown cipher mode: $self->{strMode}"); + } + + # Set read/write + $self->{bWrite} = false; + + # Create cipher object + $self->{oCipher} = new pgBackRest::LibC::Cipher::Block( + $self->{strMode} eq STORAGE_ENCRYPT ? CIPHER_MODE_ENCRYPT : CIPHER_MODE_DECRYPT, $strCipherType, $tCipherPass, + length($tCipherPass)); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# read - encrypt/decrypt data +#################################################################################################################################### +sub read +{ + my $self = shift; + my $rtBuffer = shift; + my $iSize = shift; + + # Return 0 if all data has been read + return 0 if $self->eof(); + + # Loop until required bytes have been read + my $tBufferRead = ''; + my $iBufferReadSize = 0; + + do + { + # Read data + my $tCipherBuffer; + my $iActualSize = $self->SUPER::read(\$tCipherBuffer, $iSize); + + # If something was read, then process it + if ($iActualSize > 0) + { + $tBufferRead .= $self->{oCipher}->process($tCipherBuffer); + } + + # If eof then flush the remaining data + if ($self->eof()) + { + $tBufferRead .= $self->{oCipher}->flush(); + } + + # Get the current size of the read buffer + $iBufferReadSize = length($tBufferRead); + } + while ($iBufferReadSize < $iSize && !$self->eof()); + + # Append to the read buffer + $$rtBuffer .= $tBufferRead; + + # Return the actual size read + return $iBufferReadSize; +} + +#################################################################################################################################### +# write - encrypt/decrypt data +#################################################################################################################################### +sub write +{ + my $self = shift; + my $rtBuffer = shift; + + # Set write flag so close will flush buffer + $self->{bWrite} = true; + + # Write the buffer if defined + my $tCipherBuffer; + + if (defined($$rtBuffer)) + { + $tCipherBuffer = $self->{oCipher}->process($$rtBuffer); + } + + # Call the io method. If $rtBuffer is undefined, then this is expected to error. + $self->SUPER::write(\$tCipherBuffer); + + return length($$rtBuffer); +} + +#################################################################################################################################### +# close - close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + # Only close the object if not already closed + if ($self->{oCipher}) + { + # Flush the write buffer + if ($self->{bWrite}) + { + my $tCipherBuffer = $self->{oCipher}->flush(); + $self->SUPER::write(\$tCipherBuffer); + } + + undef($self->{oCipher}); + + # Close io + return $self->SUPER::close(); + } + + return false; +} + +1; diff --git a/libc/LibC.h b/libc/LibC.h index 80c7dd01e..fed5682e2 100644 --- a/libc/LibC.h +++ b/libc/LibC.h @@ -60,3 +60,85 @@ Error handling macros that throw a Perl error when a C error is caught { \ ERROR_XS(); \ } + +/*********************************************************************************************************************************** +Core context handling macros, only intended to be called from other macros +***********************************************************************************************************************************/ +#define MEM_CONTEXT_XS_CORE_BEGIN(memContext) \ + /* Switch to the new memory context */ \ + MemContext *MEM_CONTEXT_XS_memContextOld = memContextSwitch(memContext); \ + \ + /* Store any errors to be croaked to Perl at the end */ \ + bool MEM_CONTEXT_XS_croak = false; \ + \ + /* Try the statement block */ \ + ERROR_TRY() + +#define MEM_CONTEXT_XS_CORE_END() \ + /* Set error to be croak to Perl later */ \ + ERROR_CATCH_ANY() \ + { \ + MEM_CONTEXT_XS_croak = true; \ + } \ + /* Free the context on error */ \ + ERROR_FINALLY() \ + { \ + memContextSwitch(MEM_CONTEXT_XS_memContextOld); \ + } + +/*********************************************************************************************************************************** +Simplifies creation of the memory context in contructors and includes error handling +***********************************************************************************************************************************/ +#define MEM_CONTEXT_XS_NEW_BEGIN(contextName) \ +{ \ + /* Attempt to create the memory context */ \ + MemContext *MEM_CONTEXT_XS_memContext = NULL; \ + \ + ERROR_TRY() \ + { \ + MEM_CONTEXT_XS_memContext = memContextNew(contextName); \ + } \ + ERROR_CATCH_ANY() \ + { \ + ERROR_XS() \ + } \ + \ + MEM_CONTEXT_XS_CORE_BEGIN(MEM_CONTEXT_XS_memContext) + +#define MEM_CONTEXT_XS_NEW_END() \ + MEM_CONTEXT_XS_CORE_END(); \ + \ + /* Free context and croak on error */ \ + if (MEM_CONTEXT_XS_croak) \ + { \ + memContextFree(MEM_CONTEXT_XS_memContext); \ + \ + ERROR_XS() \ + } \ +} + +#define MEM_COMTEXT_XS() \ + MEM_CONTEXT_XS_memContext + +/*********************************************************************************************************************************** +Simplifies switching the memory context in functions and includes error handling +***********************************************************************************************************************************/ +#define MEM_CONTEXT_XS_BEGIN(memContext) \ +{ \ + MEM_CONTEXT_XS_CORE_BEGIN(memContext) + +#define MEM_CONTEXT_XS_END() \ + MEM_CONTEXT_XS_CORE_END(); \ + \ + /* Croak on error */ \ + if (MEM_CONTEXT_XS_croak) \ + { \ + ERROR_XS() \ + } \ +} + +/*********************************************************************************************************************************** +Free memory context in destructors +***********************************************************************************************************************************/ +#define MEM_CONTEXT_XS_DESTROY(memContext) \ + memContextFree(memContext) diff --git a/libc/LibC.xs b/libc/LibC.xs index 6ac1ab2e5..a73f8faa9 100644 --- a/libc/LibC.xs +++ b/libc/LibC.xs @@ -25,7 +25,7 @@ C includes These includes are from the src directory. There is no Perl-specific code in them. ***********************************************************************************************************************************/ -#include "common/encode.h" +#include "cipher/random.h" #include "common/error.h" #include "config/config.h" #include "config/define.h" @@ -41,6 +41,7 @@ XSH includes These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. ***********************************************************************************************************************************/ +#include "xs/cipher/block.xsh" #include "xs/common/encode.xsh" #include "xs/config/config.auto.xsh" #include "xs/config/define.auto.xsh" @@ -68,6 +69,8 @@ INCLUDE: const-xs.inc # # These modules should map 1-1 with C modules in src directory. # ---------------------------------------------------------------------------------------------------------------------------------- +INCLUDE: xs/cipher/block.xs +INCLUDE: xs/cipher/random.xs INCLUDE: xs/common/encode.xs INCLUDE: xs/config/config.xs INCLUDE: xs/config/define.xs diff --git a/libc/Makefile.PL b/libc/Makefile.PL index c7f2f2247..2a64f504c 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -74,6 +74,8 @@ my @stryCFile = ( 'LibC.c', + 'cipher/block.c', + 'cipher/random.c', 'common/encode.c', 'common/encode/base64.c', 'common/error.c', @@ -120,5 +122,7 @@ WriteMakefile C => \@stryCFile, + LIBS => [-lcrypto], + OBJECT => '$(O_FILES)', ); diff --git a/libc/build/lib/pgBackRestLibC/Build.pm b/libc/build/lib/pgBackRestLibC/Build.pm index 9e31ffeb0..436d730a1 100644 --- a/libc/build/lib/pgBackRestLibC/Build.pm +++ b/libc/build/lib/pgBackRestLibC/Build.pm @@ -52,6 +52,14 @@ my $rhExport = )], }, + 'cipher' => + { + &BLD_EXPORTTYPE_SUB => [qw( + CIPHER_MODE_ENCRYPT + CIPHER_MODE_DECRYPT + )], + }, + 'config' => { &BLD_EXPORTTYPE_SUB => [qw( @@ -114,6 +122,13 @@ my $rhExport = encodeToStr )], }, + + 'random' => + { + &BLD_EXPORTTYPE_SUB => [qw( + randomBytes + )], + }, }; #################################################################################################################################### diff --git a/libc/lib/pgBackRest/LibCAuto.pm b/libc/lib/pgBackRest/LibCAuto.pm index 1dd1ae290..b600c490e 100644 --- a/libc/lib/pgBackRest/LibCAuto.pm +++ b/libc/lib/pgBackRest/LibCAuto.pm @@ -61,6 +61,12 @@ sub libcAutoExportTag 'pageChecksumTest', ], + cipher => + [ + 'CIPHER_MODE_ENCRYPT', + 'CIPHER_MODE_DECRYPT', + ], + config => [ 'CFGOPTVAL_INFO_OUTPUT_TEXT', @@ -236,6 +242,11 @@ sub libcAutoExportTag 'decodeToBin', 'encodeToStr', ], + + random => + [ + 'randomBytes', + ], } } diff --git a/libc/typemap b/libc/typemap new file mode 100644 index 000000000..06352cd25 --- /dev/null +++ b/libc/typemap @@ -0,0 +1 @@ +pgBackRest::LibC::Cipher::Block T_PTROBJ diff --git a/libc/xs/cipher/block.xs b/libc/xs/cipher/block.xs new file mode 100644 index 000000000..a7a7d7c30 --- /dev/null +++ b/libc/xs/cipher/block.xs @@ -0,0 +1,81 @@ +#################################################################################################################################### +# Block Cipher Perl Exports +# +# XS wrapper for functions in cipher/block.c. +#################################################################################################################################### + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Cipher::Block + +#################################################################################################################################### +pgBackRest::LibC::Cipher::Block +new(class, mode, type, key, keySize, digest = NULL) + const char *class + U32 mode + const char *type + unsigned char *key + I32 keySize + const char *digest +CODE: + RETVAL = NULL; + + // Not much point to this but it keeps the var from being unused + if (strcmp(class, PACKAGE_NAME_LIBC "::Cipher::Block") != 0) + croak("unexpected class name '%s'", class); + + MEM_CONTEXT_XS_NEW_BEGIN("cipherBlockXs") + { + RETVAL = memNew(sizeof(CipherBlockXs)); + + RETVAL->memContext = MEM_COMTEXT_XS(); + + RETVAL->pxPayload = cipherBlockNew(mode, type, key, keySize, digest); + } + MEM_CONTEXT_XS_NEW_END(); +OUTPUT: + RETVAL + +#################################################################################################################################### +SV * +process(self, svSource) + pgBackRest::LibC::Cipher::Block self + SV *svSource +CODE: + RETVAL = NULL; + STRLEN tSize; + const unsigned char *pvSource = (const unsigned char *)SvPV(svSource, tSize); + + MEM_CONTEXT_XS_BEGIN(self->memContext) + { + RETVAL = NEWSV(0, cipherBlockProcessSize(self->pxPayload, tSize)); + SvPOK_only(RETVAL); + + SvCUR_set(RETVAL, cipherBlockProcess(self->pxPayload, pvSource, tSize, (unsigned char *)SvPV_nolen(RETVAL))); + } + MEM_CONTEXT_XS_END(); +OUTPUT: + RETVAL + +#################################################################################################################################### +SV * +flush(self) + pgBackRest::LibC::Cipher::Block self +CODE: + RETVAL = NULL; + + MEM_CONTEXT_XS_BEGIN(self->memContext) + { + RETVAL = NEWSV(0, cipherBlockProcessSize(self->pxPayload, 0)); + SvPOK_only(RETVAL); + + SvCUR_set(RETVAL, cipherBlockFlush(self->pxPayload, (unsigned char *)SvPV_nolen(RETVAL))); + } + MEM_CONTEXT_XS_END(); +OUTPUT: + RETVAL + +#################################################################################################################################### +void +DESTROY(self) + pgBackRest::LibC::Cipher::Block self +CODE: + MEM_CONTEXT_XS_DESTROY(self->memContext); diff --git a/libc/xs/cipher/block.xsh b/libc/xs/cipher/block.xsh new file mode 100644 index 000000000..9464e7b07 --- /dev/null +++ b/libc/xs/cipher/block.xsh @@ -0,0 +1,15 @@ +/*********************************************************************************************************************************** +Block Cipher XS Header +***********************************************************************************************************************************/ +#include "../src/common/memContext.h" +#include "../src/cipher/block.h" + +// Encrypt/decrypt modes +#define CIPHER_MODE_ENCRYPT ((int)cipherModeEncrypt) +#define CIPHER_MODE_DECRYPT ((int)cipherModeDecrypt) + +typedef struct CipherBlockXs +{ + MemContext *memContext; + CipherBlock *pxPayload; +} CipherBlockXs, *pgBackRest__LibC__Cipher__Block; diff --git a/libc/xs/cipher/random.xs b/libc/xs/cipher/random.xs new file mode 100644 index 000000000..fe8957dde --- /dev/null +++ b/libc/xs/cipher/random.xs @@ -0,0 +1,19 @@ +# ---------------------------------------------------------------------------------------------------------------------------------- +# Random Perl Exports +# ---------------------------------------------------------------------------------------------------------------------------------- + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC + +#################################################################################################################################### +SV * +randomBytes(size) + I32 size +CODE: + RETVAL = newSV(size); + SvPOK_only(RETVAL); + + randomBytes((unsigned char *)SvPV_nolen(RETVAL), size); + + SvCUR_set(RETVAL, size); +OUTPUT: + RETVAL diff --git a/src/cipher/block.c b/src/cipher/block.c new file mode 100644 index 000000000..038dd1241 --- /dev/null +++ b/src/cipher/block.c @@ -0,0 +1,255 @@ +/*********************************************************************************************************************************** +Block Cipher +***********************************************************************************************************************************/ +#include + +#include +#include +#include + +#include "common/errorType.h" +#include "common/memContext.h" +#include "cipher/block.h" +#include "cipher/random.h" + +/*********************************************************************************************************************************** +Header constants and sizes +***********************************************************************************************************************************/ +// Magic constant for salted encrypt. Only salted encrypt is done here, but this constant is required for compatibility with the +// openssl command-line tool. +#define CIPHER_BLOCK_MAGIC "Salted__" +#define CIPHER_BLOCK_MAGIC_SIZE 8 + +// Total length of cipher header +#define CIPHER_BLOCK_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) + +/*********************************************************************************************************************************** +Track state during block encrypt/decrypt +***********************************************************************************************************************************/ +struct CipherBlock +{ + MemContext *memContext; // Context to store data + CipherMode mode; // Mode encrypt/decrypt + bool saltDone; // Has the salt been read/generated? + bool processDone; // Has any data been processed? + int passSize; // Size of passphrase in bytes + unsigned char *pass; // Passphrase used to generate encryption key + int headerSize; // Size of header read during decrypt + unsigned char header[CIPHER_BLOCK_HEADER_SIZE]; // Buffer to hold partial header during decrypt + const EVP_CIPHER *cipher; // Cipher object + const EVP_MD *digest; // Message digest object + EVP_CIPHER_CTX *cipherContext; // Encrypt/decrypt context +}; + +/*********************************************************************************************************************************** +Flag to indicate if OpenSSL has already been initialized +***********************************************************************************************************************************/ +bool openSslInitDone = false; + +/*********************************************************************************************************************************** +New block encrypt/decrypt object +***********************************************************************************************************************************/ +CipherBlock * +cipherBlockNew(CipherMode mode, const char *cipherName, const unsigned char *pass, int passSize, const char *digestName) +{ + // Only need to init once. This memory could be freed, but ciphers are used for the life of the process so don't bother. + if (!openSslInitDone) + { + ERR_load_crypto_strings(); + OpenSSL_add_all_algorithms(); + + openSslInitDone = true; + } + + // Lookup cipher by name. This means the ciphers passed in must exactly match a name expected by OpenSSL. This is a good + // thing since the name required by the openssl command-line tool will match what is used by pgBackRest. + const EVP_CIPHER *cipher = EVP_get_cipherbyname(cipherName); + + if (!cipher) + ERROR_THROW(AssertError, "unable to load cipher '%s'", cipherName); + + // Lookup digest. If not defined it will be set to sha1. + const EVP_MD *digest; + + if (digestName) + digest = EVP_get_digestbyname(digestName); + else + digest = EVP_sha1(); + + if (!digest) + ERROR_THROW(AssertError, "unable to load digest '%s'", digestName); + + // Allocate memory to hold process state + CipherBlock *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("cipherBlock") + { + // Allocate state and set context + this = memNew(sizeof(CipherBlock)); + this->memContext = MEM_CONTEXT_NEW(); + + // Set mode, encrypt or decrypt + this->mode = mode; + + // Set cipher and digest + this->cipher = cipher; + this->digest = digest; + + // Store the passphrase + this->passSize = passSize; + this->pass = memNewRaw(this->passSize); + memcpy(this->pass, pass, this->passSize); + } + MEM_CONTEXT_NEW_END(); + + return this; +} + +/*********************************************************************************************************************************** +Determine how large the destination buffer should be +***********************************************************************************************************************************/ +int +cipherBlockProcessSize(CipherBlock *this, int sourceSize) +{ + return sourceSize + EVP_MAX_BLOCK_LENGTH + CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN; +} + +/*********************************************************************************************************************************** +Encrypt/decrypt data +***********************************************************************************************************************************/ +int +cipherBlockProcess(CipherBlock *this, const unsigned char *source, int sourceSize, unsigned char *destination) +{ + // Actual destination size + uint32 destinationSize = 0; + + // If the salt has not been generated/read yet + if (!this->saltDone) + { + const unsigned char *salt = NULL; + + // On encrypt the salt is generated + if (this->mode == cipherModeEncrypt) + { + // Add magic to the destination buffer so openssl knows the file is salted + memcpy(destination, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE); + destination += CIPHER_BLOCK_MAGIC_SIZE; + destinationSize += CIPHER_BLOCK_MAGIC_SIZE; + + // Add salt to the destination buffer + randomBytes(destination, PKCS5_SALT_LEN); + salt = destination; + destination += PKCS5_SALT_LEN; + destinationSize += PKCS5_SALT_LEN; + } + // On decrypt the salt is read from the header + else + { + // Check if the entire header has been read + if (this->headerSize + sourceSize >= CIPHER_BLOCK_HEADER_SIZE) + { + // Copy header (or remains of header) from source into the header buffer + memcpy(this->header + this->headerSize, source, CIPHER_BLOCK_HEADER_SIZE - this->headerSize); + salt = this->header + CIPHER_BLOCK_MAGIC_SIZE; + + // Advance source and source size by the number of bytes read + source += CIPHER_BLOCK_HEADER_SIZE - this->headerSize; + sourceSize -= CIPHER_BLOCK_HEADER_SIZE - this->headerSize; + + // The first bytes of the file to decrypt should be equal to the magic. If not then this is not an + // encrypted file, or at least not in a format we recognize. + if (memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + ERROR_THROW(CipherError, "cipher header invalid"); + } + // Else copy what was provided into the header buffer and return 0 + else + { + memcpy(this->header + this->headerSize, source, sourceSize); + this->headerSize += sourceSize; + + // Indicate that there is nothing left to process + sourceSize = 0; + } + } + + // If salt generation/read is done + if (salt) + { + // Generate key and initialization vector + unsigned char key[EVP_MAX_KEY_LENGTH]; + unsigned char initVector[EVP_MAX_IV_LENGTH]; + + EVP_BytesToKey( + this->cipher, this->digest, salt, (unsigned char *)this->pass, this->passSize, 1, key, initVector); + + // Set free callback to ensure cipher context is freed + memContextCallback(this->memContext, (MemContextCallback)cipherBlockFree, this); + + // Create context to track cipher + if (!(this->cipherContext = EVP_CIPHER_CTX_new())) + ERROR_THROW(MemoryError, "unable to create context"); // {uncoverable - no failure path known} + + // Initialize cipher + if (EVP_CipherInit_ex( + this->cipherContext, this->cipher, NULL, key, initVector, this->mode == cipherModeEncrypt) != 1) + { + ERROR_THROW(MemoryError, "unable to initialize cipher"); // {uncoverable - no failure path known} + } + + this->saltDone = true; + } + } + + // Recheck that source size > 0 as the bytes may have been consumed reading the header + if (sourceSize > 0) + { + // Process the data + int destinationUpdateSize = 0; + + if (!EVP_CipherUpdate(this->cipherContext, destination, &destinationUpdateSize, source, sourceSize)) + ERROR_THROW(CipherError, "unable to process"); // {uncoverable - no failure path known} + + destinationSize += destinationUpdateSize; + + // Note that data has been processed so flush is valid + this->processDone = true; + } + + // Return actual destination size + return destinationSize; +} + +/*********************************************************************************************************************************** +Flush the remaining data +***********************************************************************************************************************************/ +int +cipherBlockFlush(CipherBlock *this, unsigned char *destination) +{ + // Actual destination size + int iDestinationSize = 0; + + // If no header was processed then error + if (!this->saltDone) + ERROR_THROW(CipherError, "cipher header missing"); + + // Only flush remaining data if some data was processed + if (!EVP_CipherFinal(this->cipherContext, destination, &iDestinationSize)) + ERROR_THROW(CipherError, "unable to flush"); + + // Return actual destination size + return iDestinationSize; +} + +/*********************************************************************************************************************************** +Free memory +***********************************************************************************************************************************/ +void +cipherBlockFree(CipherBlock *this) +{ + // Free cipher context + if (this->cipherContext) + EVP_CIPHER_CTX_cleanup(this->cipherContext); + + // Free mem context + memContextFree(this->memContext); +} diff --git a/src/cipher/block.h b/src/cipher/block.h new file mode 100644 index 000000000..d17581b8b --- /dev/null +++ b/src/cipher/block.h @@ -0,0 +1,20 @@ +/*********************************************************************************************************************************** +Block Cipher Header +***********************************************************************************************************************************/ +#ifndef CIPHER_BLOCK_H +#define CIPHER_BLOCK_H + +#include "cipher/cipher.h" + +// Track cipher state +typedef struct CipherBlock CipherBlock; + +// Functions +CipherBlock *cipherBlockNew( + CipherMode mode, const char *cipherName, const unsigned char *pass, int passSize, const char *digestName); +int cipherBlockProcessSize(CipherBlock *this, int sourceSize); +int cipherBlockProcess(CipherBlock *this, const unsigned char *source, int sourceSize, unsigned char *destination); +int cipherBlockFlush(CipherBlock *this, unsigned char *destination); +void cipherBlockFree(CipherBlock *this); + +#endif diff --git a/src/cipher/cipher.h b/src/cipher/cipher.h new file mode 100644 index 000000000..ae92d7e7f --- /dev/null +++ b/src/cipher/cipher.h @@ -0,0 +1,14 @@ +/*********************************************************************************************************************************** +Cipher Header +***********************************************************************************************************************************/ +#ifndef CIPHER_H +#define CIPHER_H + +// Define cipher modes +typedef enum +{ + cipherModeEncrypt, + cipherModeDecrypt, +} CipherMode; + +#endif diff --git a/src/cipher/random.c b/src/cipher/random.c new file mode 100644 index 000000000..11cbdf3bc --- /dev/null +++ b/src/cipher/random.c @@ -0,0 +1,15 @@ +/*********************************************************************************************************************************** +Cipher +***********************************************************************************************************************************/ +#include + +#include "cipher/random.h" + +/*********************************************************************************************************************************** +Generate random bytes +***********************************************************************************************************************************/ +void +randomBytes(unsigned char *buffer, int size) +{ + RAND_bytes(buffer, size); +} diff --git a/src/cipher/random.h b/src/cipher/random.h new file mode 100644 index 000000000..a9314748b --- /dev/null +++ b/src/cipher/random.h @@ -0,0 +1,10 @@ +/*********************************************************************************************************************************** +Random Header +***********************************************************************************************************************************/ +#ifndef RANDOM_H +#define RANDOM_H + +// Functions +void randomBytes(unsigned char *buffer, int size); + +#endif diff --git a/src/common/error.c b/src/common/error.c index 418d40798..c84b006f0 100644 --- a/src/common/error.c +++ b/src/common/error.c @@ -266,4 +266,4 @@ void errorInternalThrow(const ErrorType *errorType, const char *fileName, int fi // Propogate the error errorInternalPropagate(); -} // {uncoverable - errorPropagate() does not return} +} // {uncoverable - errorInternalPropagate() does not return} diff --git a/src/common/error.h b/src/common/error.h index 880daf70c..b87d86b2f 100644 --- a/src/common/error.h +++ b/src/common/error.h @@ -3,7 +3,7 @@ Error Handler Implement a try ... catch ... finally error handler. -ERROR_TRY +ERROR_TRY() { } @@ -19,7 +19,7 @@ ERROR_CATCH_ANY() { } -FINALLY +ERROR_FINALLY() { } diff --git a/src/common/errorType.c b/src/common/errorType.c index 15e046fd4..3b7b41f5f 100644 --- a/src/common/errorType.c +++ b/src/common/errorType.c @@ -32,6 +32,7 @@ ERROR_DEFINE(ERROR_CODE_MIN, AssertError, RuntimeError); ERROR_DEFINE(ERROR_CODE_MIN + 04, FormatError, RuntimeError); ERROR_DEFINE(ERROR_CODE_MIN + 69, MemoryError, RuntimeError); +ERROR_DEFINE(ERROR_CODE_MIN + 70, CipherError, FormatError); ERROR_DEFINE(ERROR_CODE_MAX, RuntimeError, RuntimeError); diff --git a/src/common/errorType.h b/src/common/errorType.h index f1b275aa9..a250df160 100644 --- a/src/common/errorType.h +++ b/src/common/errorType.h @@ -18,6 +18,7 @@ ERROR_DECLARE(AssertError); ERROR_DECLARE(FormatError); ERROR_DECLARE(MemoryError); +ERROR_DECLARE(CipherError); ERROR_DECLARE(RuntimeError); diff --git a/test/Vagrantfile b/test/Vagrantfile index 8a42dcbc3..aa9744570 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -52,7 +52,7 @@ Vagrant.configure(2) do |config| #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Build Tools' && date - apt-get install -y devscripts build-essential lintian git txt2man debhelper + apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev #--------------------------------------------------------------------------------------------------------------------------- echo 'Install AWS CLI' && date diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index 57df7a633..6f77c3df3 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -324,7 +324,7 @@ sub containerBuild " yum -y update && \\\n" . " yum -y install openssh-server openssh-clients wget sudo python-pip build-essential git \\\n" . " perl perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL \\\n" . - " gcc make perl-ExtUtils-MakeMaker perl-Test-Simple"; + " gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel"; if ($strOS eq VM_CO6) { @@ -343,7 +343,7 @@ sub containerBuild " wget --no-check-certificate -O /root/get-pip.py https://bootstrap.pypa.io/get-pip.py && \\\n" . " python /root/get-pip.py && \\\n" . " apt-get -y install openssh-server wget sudo python-pip build-essential git \\\n" . - " libdbd-pg-perl libhtml-parser-perl libio-socket-ssl-perl libxml-libxml-perl"; + " libdbd-pg-perl libhtml-parser-perl libio-socket-ssl-perl libxml-libxml-perl libssl-dev"; if ($strOS eq VM_U14) { diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index 2d2fbda8d..7417cc2b5 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -200,6 +200,35 @@ my $oTestDef = }, ] }, + # Cipher tests + { + &TESTDEF_NAME => 'cipher', + &TESTDEF_CONTAINER => true, + + &TESTDEF_TEST => + [ + { + &TESTDEF_NAME => 'random', + &TESTDEF_TOTAL => 1, + &TESTDEF_C => true, + + &TESTDEF_COVERAGE => + { + 'cipher/random' => TESTDEF_COVERAGE_FULL, + }, + }, + { + &TESTDEF_NAME => 'block', + &TESTDEF_TOTAL => 2, + &TESTDEF_C => true, + + &TESTDEF_COVERAGE => + { + 'cipher/block' => TESTDEF_COVERAGE_FULL, + }, + }, + ] + }, # PostgreSQL tests { &TESTDEF_NAME => 'postgres', @@ -283,6 +312,16 @@ my $oTestDef = &TESTDEF_TEST => [ + { + &TESTDEF_NAME => 'filter-cipher-block', + &TESTDEF_TOTAL => 2, + &TESTDEF_CLIB => true, + + &TESTDEF_COVERAGE => + { + 'Storage/Filter/CipherBlock' => TESTDEF_COVERAGE_FULL, + }, + }, { &TESTDEF_NAME => 'filter-gzip', &TESTDEF_TOTAL => 3, diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index d881b3cd8..57c82540e 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -318,7 +318,7 @@ sub run 'gcc -Wfatal-errors -std=c99 -fprofile-arcs -ftest-coverage -fPIC -O0 ' . "-I/$self->{strBackRestBase}/src -I/$self->{strBackRestBase}/test/src test.c " . "/$self->{strBackRestBase}/test/src/common/harnessTest.c " . - join(' ', @stryCFile) . ' -o test'; + join(' ', @stryCFile) . " -l crypto -o test"; executeTest( 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockTest.pm new file mode 100644 index 000000000..1486c5ff7 --- /dev/null +++ b/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockTest.pm @@ -0,0 +1,285 @@ +#################################################################################################################################### +# Tests for Block Cipher +#################################################################################################################################### +package pgBackRestTest::Module::Storage::StorageFilterCipherBlockTest; +use parent 'pgBackRestTest::Common::RunTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Fcntl qw(O_RDONLY); +use Digest::SHA qw(sha1_hex); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:random); +use pgBackRest::Storage::Base; +use pgBackRest::Storage::Filter::CipherBlock; +use pgBackRest::Storage::Posix::Driver; + +use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# run +#################################################################################################################################### +sub run +{ + my $self = shift; + + # Test data + my $strFile = $self->testPath() . qw{/} . 'file.txt'; + my $strFileEncrypt = $self->testPath() . qw{/} . 'file.enc.txt'; + my $strFileDecrypt = $self->testPath() . qw{/} . 'file.dcr.txt'; + my $strFileBin = $self->testPath() . qw{/} . 'file.bin'; + my $strFileBinEncrypt = $self->testPath() . qw{/} . 'file.enc.bin'; + my $strFileContent = 'TESTDATA'; + my $iFileLength = length($strFileContent); + my $oDriver = new pgBackRest::Storage::Posix::Driver(); + my $tCipherPass = 'areallybadkey'; + my $strCipherType = 'aes-256-cbc'; + my $tContent; + + ################################################################################################################################ + if ($self->begin('new()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + # Create an unencrypted file + executeTest("echo -n '${strFileContent}' | tee ${strFile}"); + + $self->testException( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})}, + ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS); + + $self->testException( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFile), BOGUS, $tCipherPass)}, + ERROR_ASSERT, "unable to load cipher '" . BOGUS . "'"); + + $self->testException( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openWrite($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})}, + ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS); + + $self->testException( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openWrite($strFile), BOGUS, $tCipherPass)}, + ERROR_ASSERT, "unable to load cipher '" . BOGUS . "'"); + } + + ################################################################################################################################ + if ($self->begin('read() and write()')) + { + my $tBuffer; + + #--------------------------------------------------------------------------------------------------------------------------- + # Create an plaintext file + executeTest("echo -n '${strFileContent}' | tee ${strFile}"); + + # Instantiate the cipher object - default action encrypt + my $oEncryptIo = $self->testResult(sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openRead($strFile), + $strCipherType, $tCipherPass)}, '[object]', 'new encrypt file'); + + $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (header)'); + $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (data)'); + $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes'); + + $self->testResult(sub {$tBuffer ne $strFileContent}, true, ' data read is encrypted'); + + $self->testResult(sub {$oEncryptIo->close()}, true, ' close'); + $self->testResult(sub {$oEncryptIo->close()}, false, ' close again'); + + # tBuffer is now encrypted - test write decrypts correctly + my $oDecryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileDecrypt), + $strCipherType, $tCipherPass, {strMode => STORAGE_DECRYPT})}, + '[object]', ' new decrypt file'); + + $self->testResult(sub {$oDecryptFileIo->write(\$tBuffer)}, 32, ' write decrypted'); + $self->testResult(sub {$oDecryptFileIo->close()}, true, ' close'); + + $self->testResult(sub {${$self->storageTest()->get($strFileDecrypt)}}, $strFileContent, ' data written is decrypted'); + + #--------------------------------------------------------------------------------------------------------------------------- + $tBuffer = $strFileContent; + my $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileEncrypt), + $strCipherType, $tCipherPass)}, + '[object]', 'new write encrypt'); + + $tContent = ''; + $self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, 0, ' attempt empty buffer write'); + + undef($tContent); + $self->testException( + sub {$oEncryptFileIo->write(\$tContent)}, ERROR_FILE_WRITE, + "unable to write to '${strFileEncrypt}': Use of uninitialized value"); + + # Encrypted length is not known so use tBuffer then test that tBuffer was encrypted + my $iWritten = $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, length($tBuffer), ' write encrypted'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + + $tContent = $self->storageTest()->get($strFileDecrypt); + $self->testResult(sub {defined($tContent) && $tContent ne $strFileContent}, true, ' data written is encrypted'); + + #--------------------------------------------------------------------------------------------------------------------------- + undef($tBuffer); + # Open encrypted file for decrypting + $oEncryptFileIo = + $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'new read encrypted file, decrypt'); + + # Try to read more than the length of the data expected to be output from the decrypt and confirm the decrypted length is + # the same as the original decrypted content. + $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, $iFileLength+4)}, $iFileLength, ' read all bytes'); + + # Just because length is the same does not mean content is so confirm + $self->testResult($tBuffer, $strFileContent, ' data read is decrypted'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + + #--------------------------------------------------------------------------------------------------------------------------- + undef($tContent); + undef($tBuffer); + my $strFileBinHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031'; + + executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFileBin}"); + $self->testResult( + sub {sha1_hex(${storageTest()->get($strFileBin)})}, $strFileBinHash, 'bin test - check sha1'); + + $tContent = ${storageTest()->get($strFileBin)}; + + $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openWrite($strFileBinEncrypt), $strCipherType, $tCipherPass)}, + '[object]', ' new write encrypt'); + + $self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, length($tContent), ' write encrypted'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + $self->testResult( + sub {sha1_hex(${storageTest()->get($strFileBinEncrypt)}) ne $strFileBinHash}, true, ' check sha1 different'); + + my $oEncryptBinFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileBinEncrypt), $strCipherType, $tCipherPass, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'new read encrypted bin file'); + + $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read 16777216 bytes'); + $self->testResult(sub {sha1_hex($tBuffer)}, $strFileBinHash, ' check sha1 same as original'); + $self->testResult(sub {$oEncryptBinFileIo->close()}, true, ' close'); + + # Try to read the file with the wrong passphrase + undef($tBuffer); + undef($oEncryptBinFileIo); + + $oEncryptBinFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileBinEncrypt), $strCipherType, BOGUS, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'new read Encrypted bin file with wrong passphrase'); + + $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read all bytes'); + $self->testResult(sub {sha1_hex($tBuffer) ne $strFileBinHash}, true, ' check sha1 NOT same as original'); + + # Test file against openssl to make sure they are compatible + #--------------------------------------------------------------------------------------------------------------------------- + undef($tBuffer); + + $self->storageTest()->put($strFile, $strFileContent); + + executeTest( + "openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}"); + + $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'read file encrypted by openssl'); + + $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 8, ' read 8 bytes'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + $self->testResult(sub {$tBuffer}, $strFileContent, ' check content same as original'); + + $self->storageTest()->remove($strFile); + $self->storageTest()->remove($strFileEncrypt); + + $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)}, + '[object]', 'write file to be read by openssl'); + + $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 8, ' write 8 bytes'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + + executeTest( + "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}"); + + $self->testResult(sub {${$self->storageTest()->get($strFile)}}, $strFileContent, ' check content same as original'); + + # Test empty file against openssl to make sure they are compatible + #--------------------------------------------------------------------------------------------------------------------------- + $tBuffer = ''; + + $self->storageTest()->put($strFile); + + executeTest( + "openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}"); + + $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'read empty file encrypted by openssl'); + + $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 0, ' read 0 bytes'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + $self->testResult(sub {$tBuffer}, '', ' check content same as original'); + + $self->storageTest()->remove($strFile); + $self->storageTest()->remove($strFileEncrypt); + + $oEncryptFileIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)}, + '[object]', 'write file to be read by openssl'); + + $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 0, ' write 0 bytes'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); + + executeTest( + "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}"); + + $self->testResult(sub {${$self->storageTest()->get($strFile)}}, undef, ' check content same as original'); + + # Error on empty file decrypt - an empty file that has been encrypted will be 32 bytes + #--------------------------------------------------------------------------------------------------------------------------- + undef($tBuffer); + $self->storageTest()->put($strFileEncrypt); + + $oEncryptFileIo = + $self->testResult( + sub {new pgBackRest::Storage::Filter::CipherBlock( + $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, + {strMode => STORAGE_DECRYPT})}, + '[object]', 'new read empty attempt decrypt'); + + $self->testException(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, ERROR_CIPHER, 'cipher header missing'); + $self->testResult(sub {$oEncryptFileIo->close()}, true, 'close'); + + # OpenSSL should error on the empty file + executeTest( + "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}", + {iExpectedExitStatus => 1}); + } +} + +1; diff --git a/test/src/module/cipher/blockTest.c b/test/src/module/cipher/blockTest.c new file mode 100644 index 000000000..904c2f5d8 --- /dev/null +++ b/test/src/module/cipher/blockTest.c @@ -0,0 +1,180 @@ +/*********************************************************************************************************************************** +Test Block Cipher +***********************************************************************************************************************************/ +#include + +/*********************************************************************************************************************************** +Data for testing +***********************************************************************************************************************************/ +#define TEST_CIPHER "aes-256-cbc" +#define TEST_PASS "areallybadpassphrase" +#define TEST_PASS_SIZE strlen(TEST_PASS) +#define TEST_PLAINTEXT "plaintext" +#define TEST_DIGEST "sha256" +#define TEST_BUFFER_SIZE 256 + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void testRun() +{ + // ----------------------------------------------------------------------------------------------------------------------------- + if (testBegin("blockCipherNew() and blockCipherFree()")) + { + // Cipher and digest errors + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ERROR( + cipherBlockNew( + cipherModeEncrypt, BOGUS_STR, TEST_PASS, TEST_PASS_SIZE, NULL), AssertError, "unable to load cipher 'BOGUS'"); + TEST_ERROR( + cipherBlockNew(cipherModeEncrypt, NULL, TEST_PASS, TEST_PASS_SIZE, NULL), AssertError, + "unable to load cipher '(null)'"); + TEST_ERROR( + cipherBlockNew( + cipherModeEncrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, BOGUS_STR), AssertError, + "unable to load digest 'BOGUS'"); + + // Initialization of object + // ------------------------------------------------------------------------------------------------------------------------- + CipherBlock *cipherBlock = cipherBlockNew(cipherModeEncrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + TEST_RESULT_STR(memContextName(cipherBlock->memContext), "cipherBlock", "mem context name is valid"); + TEST_RESULT_INT(cipherBlock->mode, cipherModeEncrypt, "mode is valid"); + TEST_RESULT_INT(cipherBlock->passSize, TEST_PASS_SIZE, "passphrase size is valid"); + TEST_RESULT_BOOL(memcmp(cipherBlock->pass, TEST_PASS, TEST_PASS_SIZE) == 0, true, "passphrase is valid"); + TEST_RESULT_BOOL(cipherBlock->saltDone, false, "salt done is false"); + TEST_RESULT_BOOL(cipherBlock->processDone, false, "process done is false"); + TEST_RESULT_INT(cipherBlock->headerSize, 0, "header size is 0"); + TEST_RESULT_PTR_NE(cipherBlock->cipher, NULL, "cipher is set"); + TEST_RESULT_PTR_NE(cipherBlock->digest, NULL, "digest is set"); + TEST_RESULT_PTR(cipherBlock->cipherContext, NULL, "cipher context is not set"); + memContextFree(cipherBlock->memContext); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (testBegin("Encrypt and Decrypt")) + { + char encryptBuffer[TEST_BUFFER_SIZE]; + int encryptSize = 0; + char decryptBuffer[TEST_BUFFER_SIZE]; + int decryptSize = 0; + + // Encrypt + // ------------------------------------------------------------------------------------------------------------------------- + CipherBlock *blockEncrypt = cipherBlockNew(cipherModeEncrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + encryptSize = cipherBlockProcess(blockEncrypt, TEST_PLAINTEXT, strlen(TEST_PLAINTEXT), encryptBuffer); + + TEST_RESULT_BOOL(blockEncrypt->saltDone, true, "salt done is true"); + TEST_RESULT_BOOL(blockEncrypt->processDone, true, "process done is true"); + TEST_RESULT_INT(blockEncrypt->headerSize, 0, "header size is 0"); + TEST_RESULT_INT(encryptSize, CIPHER_BLOCK_HEADER_SIZE, "cipher size is header len"); + + TEST_RESULT_INT( + cipherBlockProcessSize(blockEncrypt, strlen(TEST_PLAINTEXT)), + strlen(TEST_PLAINTEXT) + EVP_MAX_BLOCK_LENGTH + CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN, "check process size"); + + encryptSize += cipherBlockProcess(blockEncrypt, TEST_PLAINTEXT, strlen(TEST_PLAINTEXT), encryptBuffer + encryptSize); + TEST_RESULT_INT( + encryptSize, CIPHER_BLOCK_HEADER_SIZE + EVP_CIPHER_block_size(blockEncrypt->cipher), + "cipher size increases by one block"); + + encryptSize += cipherBlockFlush(blockEncrypt, encryptBuffer + encryptSize); + TEST_RESULT_INT( + encryptSize, CIPHER_BLOCK_HEADER_SIZE + (EVP_CIPHER_block_size(blockEncrypt->cipher) * 2), + "cipher size increases by one block on flush"); + + cipherBlockFree(blockEncrypt); + + // Decrypt in one pass + // ------------------------------------------------------------------------------------------------------------------------- + CipherBlock *blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + decryptSize = cipherBlockProcess(blockDecrypt, encryptBuffer, encryptSize, decryptBuffer); + TEST_RESULT_INT(decryptSize, EVP_CIPHER_block_size(blockDecrypt->cipher), "decrypt size is one block"); + + decryptSize += cipherBlockFlush(blockDecrypt, decryptBuffer + decryptSize); + TEST_RESULT_INT(decryptSize, strlen(TEST_PLAINTEXT) * 2, "check final decrypt size"); + + decryptBuffer[decryptSize] = 0; + TEST_RESULT_STR(decryptBuffer, (TEST_PLAINTEXT TEST_PLAINTEXT), "check final decrypt buffer"); + + // Decrypt in small chunks to test buffering + // ------------------------------------------------------------------------------------------------------------------------- + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + decryptSize = 0; + memset(decryptBuffer, 0, TEST_BUFFER_SIZE); + + decryptSize = cipherBlockProcess(blockDecrypt, encryptBuffer, CIPHER_BLOCK_MAGIC_SIZE, decryptBuffer); + TEST_RESULT_INT(decryptSize, 0, "no decrypt since header read is not complete"); + TEST_RESULT_BOOL(blockDecrypt->saltDone, false, "salt done is false"); + TEST_RESULT_BOOL(blockDecrypt->processDone, false, "process done is false"); + TEST_RESULT_INT(blockDecrypt->headerSize, CIPHER_BLOCK_MAGIC_SIZE, "check header size"); + TEST_RESULT_BOOL( + memcmp(blockDecrypt->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) == 0, true, "check header magic"); + + decryptSize += cipherBlockProcess( + blockDecrypt, encryptBuffer + CIPHER_BLOCK_MAGIC_SIZE, PKCS5_SALT_LEN, decryptBuffer + decryptSize); + TEST_RESULT_INT(decryptSize, 0, "no decrypt since no data processed yet"); + TEST_RESULT_BOOL(blockDecrypt->saltDone, true, "salt done is true"); + TEST_RESULT_BOOL(blockDecrypt->processDone, false, "process done is false"); + TEST_RESULT_INT(blockDecrypt->headerSize, CIPHER_BLOCK_MAGIC_SIZE, "check header size (not increased)"); + TEST_RESULT_BOOL( + memcmp( + blockDecrypt->header + CIPHER_BLOCK_MAGIC_SIZE, encryptBuffer + CIPHER_BLOCK_MAGIC_SIZE, + PKCS5_SALT_LEN) == 0, + true, "check header salt"); + + decryptSize += cipherBlockProcess( + blockDecrypt, encryptBuffer + CIPHER_BLOCK_HEADER_SIZE, encryptSize - CIPHER_BLOCK_HEADER_SIZE, + decryptBuffer + decryptSize); + TEST_RESULT_INT(decryptSize, EVP_CIPHER_block_size(blockDecrypt->cipher), "decrypt size is one block"); + + decryptSize += cipherBlockFlush(blockDecrypt, decryptBuffer + decryptSize); + TEST_RESULT_INT(decryptSize, strlen(TEST_PLAINTEXT) * 2, "check final decrypt size"); + + decryptBuffer[decryptSize] = 0; + TEST_RESULT_STR(decryptBuffer, (TEST_PLAINTEXT TEST_PLAINTEXT), "check final decrypt buffer"); + + // Encrypt zero byte file and decrypt it + // ------------------------------------------------------------------------------------------------------------------------- + blockEncrypt = cipherBlockNew(cipherModeEncrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_RESULT_INT(cipherBlockProcess(blockEncrypt, NULL, 0, encryptBuffer), 16, "process header"); + TEST_RESULT_INT(cipherBlockFlush(blockEncrypt, encryptBuffer + 16), 16, "flush remaining bytes"); + + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_RESULT_INT(cipherBlockProcess(blockDecrypt, encryptBuffer, 32, decryptBuffer), 0, "0 bytes processed"); + TEST_RESULT_INT(cipherBlockFlush(blockDecrypt, decryptBuffer), 0, "0 bytes on flush"); + + // Invalid cipher header + // ------------------------------------------------------------------------------------------------------------------------- + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_ERROR(cipherBlockProcess(blockDecrypt, "1234567890123456", 16, decryptBuffer), CipherError, "cipher header invalid"); + + // Invalid encrypted data cannot be flushed + // ------------------------------------------------------------------------------------------------------------------------- + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_RESULT_INT(cipherBlockProcess(blockDecrypt, CIPHER_BLOCK_MAGIC "12345678", 16, decryptBuffer), 0, "process header"); + TEST_RESULT_INT(cipherBlockProcess(blockDecrypt, "1234567890123456", 16, decryptBuffer), 0, "process 0 bytes"); + + TEST_ERROR(cipherBlockFlush(blockDecrypt, decryptBuffer), CipherError, "unable to flush"); + + // File with no header should not flush + // ------------------------------------------------------------------------------------------------------------------------- + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_RESULT_INT(cipherBlockProcess(blockDecrypt, NULL, 0, decryptBuffer), 0, "no header processed"); + TEST_ERROR(cipherBlockFlush(blockDecrypt, decryptBuffer), CipherError, "cipher header missing"); + + // File with header only should error + // ------------------------------------------------------------------------------------------------------------------------- + blockDecrypt = cipherBlockNew(cipherModeDecrypt, TEST_CIPHER, TEST_PASS, TEST_PASS_SIZE, NULL); + + TEST_RESULT_INT(cipherBlockProcess(blockDecrypt, CIPHER_BLOCK_MAGIC "12345678", 16, decryptBuffer), 0, "0 bytes processed"); + TEST_ERROR(cipherBlockFlush(blockDecrypt, decryptBuffer), CipherError, "unable to flush"); + } +} diff --git a/test/src/module/cipher/randomTest.c b/test/src/module/cipher/randomTest.c new file mode 100644 index 000000000..55904c00e --- /dev/null +++ b/test/src/module/cipher/randomTest.c @@ -0,0 +1,32 @@ +/*********************************************************************************************************************************** +Test Random +***********************************************************************************************************************************/ +#include "common/memContext.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void testRun() +{ + // ----------------------------------------------------------------------------------------------------------------------------- + if (testBegin("randomBytes()")) + { + // ------------------------------------------------------------------------------------------------------------------------- + // Test if the buffer was overrun + int bufferSize = 256; + char *buffer = memNew(bufferSize); + + randomBytes(buffer, bufferSize); + TEST_RESULT_BOOL(buffer[bufferSize] == 0, true, "check that buffer did not overrun (though random byte could be 0)"); + + // ------------------------------------------------------------------------------------------------------------------------- + // Count bytes that are not zero (there shouldn't be all zeroes) + int nonZeroTotal = 0; + + for (int charIdx = 0; charIdx < bufferSize; charIdx++) + if (buffer[charIdx] != 0) + nonZeroTotal++; + + TEST_RESULT_INT_NE(nonZeroTotal, 0, "check that there are non-zero values in the buffer"); + } +}