From 59f135340d4f76c522c8b24ecb23d56b57b2b0f8 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 25 Jul 2019 14:34:16 -0400 Subject: [PATCH] The local command for backup is implemented entirely in C. The local process is now entirely migrated to C. Since all major I/O operations are performed in the local process, the vast majority of I/O is now performed in C. Contributed by David Steele, Cynthia Shang. --- doc/xml/release.xml | 11 + lib/pgBackRest/Backup/Backup.pm | 22 +- lib/pgBackRest/Backup/File.pm | 244 +--------- lib/pgBackRest/Main.pm | 24 - lib/pgBackRest/Protocol/Local/Minion.pm | 69 --- libc/Makefile.PL | 1 - libc/xs/storage/storage.xsh | 17 - src/Makefile.in | 12 +- src/command/backup/file.c | 252 ++++++++++ src/command/backup/file.h | 48 ++ src/command/backup/protocol.c | 67 +++ src/command/backup/protocol.h | 22 + src/command/local/local.c | 2 + src/main.c | 10 +- src/perl/embed.auto.c | 311 +----------- test/define.yaml | 9 + .../Module/Backup/BackupFileUnitPerlTest.pm | 329 +++---------- test/src/module/command/backupTest.c | 457 ++++++++++++++++++ 18 files changed, 990 insertions(+), 917 deletions(-) delete mode 100644 lib/pgBackRest/Protocol/Local/Minion.pm create mode 100644 src/command/backup/file.c create mode 100644 src/command/backup/file.h create mode 100644 src/command/backup/protocol.c create mode 100644 src/command/backup/protocol.h create mode 100644 test/src/module/command/backupTest.c diff --git a/doc/xml/release.xml b/doc/xml/release.xml index cb01200e5..9e5c1e855 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -33,6 +33,17 @@ + + + + + + + +

The local command for backup is implemented entirely in C.

+ + + diff --git a/lib/pgBackRest/Backup/Backup.pm b/lib/pgBackRest/Backup/Backup.pm index 6854a2669..b6bc3f8db 100644 --- a/lib/pgBackRest/Backup/Backup.pm +++ b/lib/pgBackRest/Backup/Backup.pm @@ -336,13 +336,6 @@ sub processManifest } } - # Build the lsn start parameter to pass to the extra function - my $hStartLsnParam = - { - iWalId => defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF, - iWalOffset => defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF, - }; - # Iterate all files in the manifest foreach my $strRepoFile ( sort {sprintf("%016d-%s", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $b, MANIFEST_SUBKEY_SIZE), $b) cmp @@ -401,13 +394,13 @@ sub processManifest # Queue for parallel backup $oBackupProcess->queueJob( $iHostConfigIdx, $strQueueKey, $strRepoFile, OP_BACKUP_FILE, - [$strDbFile, $strRepoFile, $lSize, + [$strDbFile, $bIgnoreMissing, $lSize, $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, false), - cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $strBackupLabel, $bCompress, - cfgOption(CFGOPT_COMPRESS_LEVEL), $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, - MANIFEST_SUBKEY_TIMESTAMP, false), $bIgnoreMissing, - cfgOption(CFGOPT_CHECKSUM_PAGE) && isChecksumPage($strRepoFile) ? $hStartLsnParam : undef, - cfgOption(CFGOPT_DELTA), defined($strReference) ? true : false], + cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, + defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF, + defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF, + $strRepoFile, defined($strReference) ? true : false, $bCompress, cfgOption(CFGOPT_COMPRESS_LEVEL), + $strBackupLabel, cfgOption(CFGOPT_DELTA)], {rParamSecure => $oBackupManifest->cipherPassSub() ? [$oBackupManifest->cipherPassSub()] : undef}); # Size and checksum will be removed and then verified later as a sanity check @@ -449,7 +442,8 @@ sub processManifest { ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $hJob->{iHostConfigIdx}), false), - $hJob->{iProcessId}, @{$hJob->{rParam}}[0..4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize, + $hJob->{iProcessId}, @{$hJob->{rParam}}[0], @{$hJob->{rParam}}[7], @{$hJob->{rParam}}[2], @{$hJob->{rParam}}[3], + @{$hJob->{rParam}}[4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize, $lManifestSaveCurrent); } diff --git a/lib/pgBackRest/Backup/File.pm b/lib/pgBackRest/Backup/File.pm index 87167b3aa..e2325b37d 100644 --- a/lib/pgBackRest/Backup/File.pm +++ b/lib/pgBackRest/Backup/File.pm @@ -37,235 +37,6 @@ use constant BACKUP_FILE_SKIP => 3; use constant BACKUP_FILE_NOOP => 4; push @EXPORT, qw(BACKUP_FILE_NOOP); -#################################################################################################################################### -# backupFile -#################################################################################################################################### -sub backupFile -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbFile, # Database file to backup - $strRepoFile, # Location in the repository to copy to - $lSizeFile, # File size - $strChecksum, # File checksum to be checked - $bChecksumPage, # Should page checksums be calculated? - $strBackupLabel, # Label of current backup - $bCompress, # Compress destination file - $iCompressLevel, # Compress level - $lModificationTime, # File modification time - $bIgnoreMissing, # Is it OK if the file is missing? - $hExtraParam, # Parameter to pass to the extra function - $bDelta, # Is the delta option on? - $bHasReference, # Does the file exist in the repo in a prior backup in the set? - $strCipherPass, # Passphrase to access the repo file (undefined if repo not encrypted). This - # parameter must always be last in the parameter list to this function. - ) = - logDebugParam - ( - __PACKAGE__ . '::backupFile', \@_, - {name => 'strDbFile', trace => true}, - {name => 'strRepoFile', trace => true}, - {name => 'lSizeFile', trace => true}, - {name => 'strChecksum', required => false, trace => true}, - {name => 'bChecksumPage', trace => true}, - {name => 'strBackupLabel', trace => true}, - {name => 'bCompress', trace => true}, - {name => 'iCompressLevel', trace => true}, - {name => 'lModificationTime', trace => true}, - {name => 'bIgnoreMissing', default => true, trace => true}, - {name => 'hExtraParam', required => false, trace => true}, - {name => 'bDelta', trace => true}, - {name => 'bHasReference', trace => true}, - {name => 'strCipherPass', required => false, trace => true}, - ); - - my $oStorageRepo = storageRepo(); # Repo storage - my $iCopyResult = BACKUP_FILE_COPY; # Copy result - my $strCopyChecksum; # Copy checksum - my $rExtra; # Page checksum result - my $lCopySize; # Copy Size - my $lRepoSize; # Repo size - - # Add compression suffix if needed - my $strFileOp = $strRepoFile . ($bCompress ? '.' . COMPRESS_EXT : ''); - - my $bCopy = true; - - # If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo, else just - # check the repo. - if (defined($strChecksum)) - { - # If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case then recopy. - if ($bDelta) - { - ($strCopyChecksum, $lCopySize) = storageDb()->hashSize($strDbFile, {bIgnoreMissing => $bIgnoreMissing}); - - # If the DB file exists, then check the checksum - if (defined($strCopyChecksum)) - { - $bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile); - - # If the database file checksum and size are same and the file is in a prior backup, then no need to copy. If the - # checksum/size do not match, that is OK, just leave the copy result as COPY so the file will be copied to this - # backup. - if (!$bCopy && $bHasReference) - { - $iCopyResult = BACKUP_FILE_NOOP; - } - } - # Else the source file is missing from the database so skip this file - else - { - $iCopyResult = BACKUP_FILE_SKIP; - $bCopy = false; - } - } - - # If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also test the - # checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, then there may be - # corruption in the repo, so recopy - if (!$bDelta || !$bHasReference) - { - # If this is a delta backup and the file is missing from the DB, then remove it from the repo (backupManifestUpdate will - # remove it from the manifest) - if ($iCopyResult == BACKUP_FILE_SKIP) - { - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"); - } - elsif (!$bDelta || !$bCopy) - { - # Add decompression - my $rhyFilter; - - if ($bCompress) - { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}); - } - - # Get the checksum - ($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize( - $oStorageRepo->openRead(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}", - {rhyFilter => $rhyFilter, strCipherPass => $strCipherPass})); - - # Determine if the file needs to be recopied - $bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile); - - # Set copy result - $iCopyResult = $bCopy ? BACKUP_FILE_RECOPY : BACKUP_FILE_CHECKSUM; - } - } - } - - # Copy the file - if ($bCopy) - { - # Add size and sha filters - my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}]; - - # Add page checksum filter - if ($bChecksumPage) - { - # Determine which segment no this is by checking for a numeric extension. No extension means segment 0. - my $iSegmentNo = ($strDbFile =~ /\.[0-9]+$/) ? substr(($strDbFile =~ m/\.[0-9]+$/g)[0], 1) + 0 : 0; - - push( - @{$rhyFilter}, - {strClass => "pgBackRest::Backup::Filter::PageChecksum", - rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]}); - }; - - # Add compression - if ($bCompress) - { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]}); - } - # Else add protocol compression if the destination is not compressed and there is no encryption - elsif (!defined($strCipherPass)) - { - push( - @{$rhyFilter}, - {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]}); - } - - # Open the source file - my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing}); - - # Open the destination file - $rhyFilter = undef; - - # Add protocol decompression if the destination is not compressed and there is no encryption - if (!$bCompress && !defined($strCipherPass)) - { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]}); - } - - my $oDestinationFileIo = $oStorageRepo->openWrite( - STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}", - {bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass}); - - $oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef); - - # Copy the file - if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo)) - { - # Get sha checksum and size - $strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA); - $lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE); - $lRepoSize = $oDestinationFileIo->result(COMMON_IO_HANDLE); - - if (!defined($lRepoSize)) - { - confess &log(ERROR, "REPO_SIZE IS NOT SET"); - } - - # Get results of page checksum validation - if ($bChecksumPage) - { - my $rExtraRaw = $oSourceFileIo->result("pgBackRest::Backup::Filter::PageChecksum"); - - $rExtra = - { - bValid => $rExtraRaw->{valid} ? true : false, - bAlign => $rExtraRaw->{align} ? true : false, - iyPageError => $rExtraRaw->{error}, - }; - } - } - # Else if source file is missing the database removed it - else - { - $iCopyResult = BACKUP_FILE_SKIP; - } - } - - # If the file was copied get the repo size only if the storage can store the files with a different size than what was written. - # This has to be checked after the file is at rest because filesystem compression may affect the actual repo size and this - # cannot be calculated in stream. - # - # If the file was checksummed then get the size in all cases since we don't already have it. - if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) && - $oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) || - $iCopyResult == BACKUP_FILE_CHECKSUM) - { - $lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->{size}; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iCopyResult', value => $iCopyResult}, - {name => 'lCopySize', value => $lCopySize}, - {name => 'lRepoSize', value => $lRepoSize}, - {name => 'strCopyChecksum', value => $strCopyChecksum}, - {name => 'rExtra', value => $rExtra}, - ); -} - -push @EXPORT, qw(backupFile); - #################################################################################################################################### # backupManifestUpdate #################################################################################################################################### @@ -384,21 +155,22 @@ sub backupManifestUpdate if ($bChecksumPage) { # The valid flag should be set - if (defined($rExtra->{bValid})) + if (defined($rExtra->{valid})) { # Store the valid flag - $oManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{bValid}); + $oManifest->boolSet( + MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{valid}); # If the page was not valid - if (!$rExtra->{bValid}) + if (!$rExtra->{valid}) { # Check for a page misalignment if ($lSizeCopy % PG_PAGE_SIZE != 0) { # Make sure the align flag was set, otherwise there is a bug - if (!defined($rExtra->{bAlign}) || $rExtra->{bAlign}) + if (!defined($rExtra->{align}) || $rExtra->{align}) { - confess &log(ASSERT, 'bAlign flag should have been set for misaligned page'); + confess &log(ASSERT, 'align flag should have been set for misaligned page'); } # Emit a warning so the user knows something is amiss @@ -411,13 +183,13 @@ sub backupManifestUpdate { $oManifest->set( MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - dclone($rExtra->{iyPageError})); + dclone($rExtra->{error})); # Build a pretty list of the page errors my $strPageError; my $iPageErrorTotal = 0; - foreach my $iyPage (@{$rExtra->{iyPageError}}) + foreach my $iyPage (@{$rExtra->{error}}) { $strPageError .= (defined($strPageError) ? ', ' : ''); diff --git a/lib/pgBackRest/Main.pm b/lib/pgBackRest/Main.pm index 52f7273f4..ba2083d8f 100644 --- a/lib/pgBackRest/Main.pm +++ b/lib/pgBackRest/Main.pm @@ -107,30 +107,6 @@ sub main cfgOption(CFGOPT_LOCK_PATH), cfgOption(CFGOPT_COMMAND), cfgOption(CFGOPT_STANZA, false), cfgOption(CFGOPT_PROCESS)); } - # Process local command - # -------------------------------------------------------------------------------------------------------------------------- - elsif (cfgCommandTest(CFGCMD_LOCAL)) - { - # Set log levels - cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true); - logLevelSet(cfgOption(CFGOPT_LOG_LEVEL_FILE), OFF, cfgOption(CFGOPT_LOG_LEVEL_STDERR)); - - logFileSet( - storageLocal(), - cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgOption(CFGOPT_COMMAND)) . '-' . - lc(cfgCommandName(cfgCommandGet())) . '-' . sprintf("%03d", cfgOption(CFGOPT_PROCESS))); - - # Load module dynamically - require pgBackRest::Protocol::Local::Minion; - pgBackRest::Protocol::Local::Minion->import(); - - # Create the local object - my $oLocal = new pgBackRest::Protocol::Local::Minion(); - - # Process local requests - $oLocal->process(); - } - # Process check command # -------------------------------------------------------------------------------------------------------------------------- elsif (cfgCommandTest(CFGCMD_CHECK)) diff --git a/lib/pgBackRest/Protocol/Local/Minion.pm b/lib/pgBackRest/Protocol/Local/Minion.pm deleted file mode 100644 index 596336b51..000000000 --- a/lib/pgBackRest/Protocol/Local/Minion.pm +++ /dev/null @@ -1,69 +0,0 @@ -#################################################################################################################################### -# PROTOCOL LOCAL MINION MODULE -#################################################################################################################################### -package pgBackRest::Protocol::Local::Minion; -use parent 'pgBackRest::Protocol::Command::Minion'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use pgBackRest::Backup::File; -use pgBackRest::Common::Log; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Base::Master; -use pgBackRest::Protocol::Base::Minion; -use pgBackRest::Protocol::Command::Minion; -use pgBackRest::Protocol::Helper; -use pgBackRest::RestoreFile; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->new'); - - # Init object and store variables - my $self = $class->SUPER::new(cfgCommandName(CFGCMD_LOCAL), cfgOption(CFGOPT_BUFFER_SIZE), cfgOption(CFGOPT_PROTOCOL_TIMEOUT)); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# init -#################################################################################################################################### -sub init -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->init'); - - # Create anonymous subs for each command - my $hCommandMap = - { - &OP_BACKUP_FILE => sub {backupFile(@{shift()})}, - - # To be run after each command to keep the remote alive - &OP_POST => sub {protocolKeepAlive()}, - }; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hCommandMap', value => $hCommandMap} - ); -} - -1; diff --git a/libc/Makefile.PL b/libc/Makefile.PL index 8bedd5fb0..e67c4c3c5 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -42,7 +42,6 @@ my @stryCFile = ( 'LibC.c', - 'command/backup/pageChecksum.c', 'command/command.c', 'common/compress/gzip/common.c', 'common/compress/gzip/compress.c', diff --git a/libc/xs/storage/storage.xsh b/libc/xs/storage/storage.xsh index 4f4dde431..7229b29cf 100644 --- a/libc/xs/storage/storage.xsh +++ b/libc/xs/storage/storage.xsh @@ -1,7 +1,6 @@ /*********************************************************************************************************************************** Storage XS Header ***********************************************************************************************************************************/ -#include "command/backup/pageChecksum.h" #include "common/assert.h" #include "common/compress/gzip/compress.h" #include "common/compress/gzip/decompress.h" @@ -127,14 +126,6 @@ storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const Strin { ioFilterGroupAdd(filterGroup, ioSizeNew()); } - else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum")) - { - ioFilterGroupAdd( - filterGroup, - pageChecksumNew( - varUInt64Force(varLstGet(paramList, 0)), PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, - (uint64_t)varUIntForce(varLstGet(paramList, 1)) << 32 | varUIntForce(varLstGet(paramList, 2)))); - } else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gzip")) { if (strEqZ(varStr(varLstGet(paramList, 0)), "compress")) @@ -167,10 +158,6 @@ storageFilterXsResult(const IoFilterGroup *filterGroup, const String *filter) { result = ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR); } - else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum")) - { - result = ioFilterGroupResult(filterGroup, PAGE_CHECKSUM_FILTER_TYPE_STR); - } else THROW_FMT(AssertError, "unable to get result for invalid filter '%s'", strPtr(filter)); @@ -202,10 +189,6 @@ storageFilterXsResultAll(const IoFilterGroup *filterGroup) { filterPerl = strNew("pgBackRest::Common::Io::Handle"); } - else if (strEq(filter, PAGE_CHECKSUM_FILTER_TYPE_STR)) - { - filterPerl = strNew("pgBackRest::Backup::Filter::PageChecksum"); - } if (filterPerl != NULL) { diff --git a/src/Makefile.in b/src/Makefile.in index 451ff0ab4..a78fd3b9d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -52,7 +52,9 @@ SRCS = \ command/archive/push/protocol.c \ command/archive/push/push.c \ command/backup/common.c \ + command/backup/file.c \ command/backup/pageChecksum.c \ + command/backup/protocol.c \ command/expire/expire.c \ command/help/help.c \ command/info/info.c \ @@ -219,9 +221,15 @@ command/archive/push/push.o: command/archive/push/push.c build.auto.h command/ar command/backup/common.o: command/backup/common.c build.auto.h command/backup/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/common.c -o command/backup/common.o +command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/common.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/file.c -o command/backup/file.o + command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h command/backup/pageChecksum.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/pageChecksum.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/pageChecksum.c -o command/backup/pageChecksum.o +command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/protocol.c -o command/backup/protocol.o + command/command.o: command/command.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/tls/client.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/command.c -o command/command.o @@ -237,7 +245,7 @@ command/help/help.o: command/help/help.c build.auto.h common/assert.h common/deb command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/info/info.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o -command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h +command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/backup/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/local/local.c -o command/local/local.o command/remote/remote.o: command/remote/remote.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/remote/protocol.h @@ -438,7 +446,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push. perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o -perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh +perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o postgres/interface.o: postgres/interface.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/version.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h diff --git a/src/command/backup/file.c b/src/command/backup/file.c new file mode 100644 index 000000000..de96717ad --- /dev/null +++ b/src/command/backup/file.c @@ -0,0 +1,252 @@ +/*********************************************************************************************************************************** +Backup File +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "command/backup/file.h" +#include "command/backup/pageChecksum.h" +#include "common/compress/gzip/common.h" +#include "common/compress/gzip/compress.h" +#include "common/compress/gzip/decompress.h" +#include "common/crypto/cipherBlock.h" +#include "common/crypto/hash.h" +#include "common/debug.h" +#include "common/io/filter/group.h" +#include "common/io/filter/size.h" +#include "common/io/io.h" +#include "common/log.h" +#include "common/regExp.h" +#include "common/type/convert.h" +#include "postgres/interface.h" +#include "storage/helper.h" + +/*********************************************************************************************************************************** +Helper functions +***********************************************************************************************************************************/ +static unsigned int +segmentNumber(const String *pgFile) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, pgFile); + FUNCTION_TEST_END(); + + // Determine which segment number this is by checking for a numeric extension. No extension means segment 0. + FUNCTION_TEST_RETURN(regExpMatchOne(STRDEF("\\.[0-9]+$"), pgFile) ? cvtZToUInt(strrchr(strPtr(pgFile), '.') + 1) : 0); +} + +/*********************************************************************************************************************************** +Copy a file from the PostgreSQL data directory to the repository +***********************************************************************************************************************************/ +BackupFileResult +backupFile( + const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage, + uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress, + unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STRING, pgFile); // Database file to copy to the repo + FUNCTION_LOG_PARAM(BOOL, pgFileIgnoreMissing); // Is it OK if the database file is missing? + FUNCTION_LOG_PARAM(UINT64, pgFileSize); // Size of the database file + FUNCTION_LOG_PARAM(STRING, pgFileChecksum); // Checksum to verify the database file + FUNCTION_LOG_PARAM(BOOL, pgFileChecksumPage); // Should page checksums be validated + FUNCTION_LOG_PARAM(UINT64, pgFileChecksumPageLsnLimit); // Upper LSN limit to which page checksums must be valid + FUNCTION_LOG_PARAM(STRING, repoFile); // Destination in the repo to copy the pg file + FUNCTION_LOG_PARAM(BOOL, repoFileHasReference); // Does the repo file exists in a prior backup in the set? + FUNCTION_LOG_PARAM(BOOL, repoFileCompress); // Compress destination file + FUNCTION_LOG_PARAM(UINT, repoFileCompressLevel); // Compression level for destination file + FUNCTION_LOG_PARAM(STRING, backupLabel); // Label of current backup + FUNCTION_LOG_PARAM(BOOL, delta); // Is the delta option on? + FUNCTION_LOG_PARAM(ENUM, cipherType); // Encryption type + FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted + FUNCTION_LOG_END(); + + ASSERT(pgFile != NULL); + ASSERT(repoFile != NULL); + ASSERT(backupLabel != NULL); + ASSERT((cipherType == cipherTypeNone && cipherPass == NULL) || (cipherType != cipherTypeNone && cipherPass != NULL)); + + // Backup file results + BackupFileResult result = {.backupCopyResult = backupCopyResultCopy}; + + MEM_CONTEXT_TEMP_BEGIN() + { + // Generate complete repo path and add compression extension if needed + const String *repoPathFile = strNewFmt( + STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(backupLabel), strPtr(repoFile), repoFileCompress ? "." GZIP_EXT : ""); + + // If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo, else + // just check the repo. + if (pgFileChecksum != NULL) + { + // Does the file in pg match the checksum and size passed? + bool pgFileMatch = false; + + // If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case then + // recopy. + if (delta) + { + // Generate checksum/size for the pg file + IoRead *read = storageReadIo(storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing)); + ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR)); + ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew()); + + // If the pg file exists check the checksum/size + if (ioReadDrain(read)) + { + const String *pgTestChecksum = varStr( + ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR)); + uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR)); + + // Does the pg file match? + if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum)) + { + pgFileMatch = true; + + // If it matches and is a reference to a previous backup then no need to copy the file + if (repoFileHasReference) + { + memContextSwitch(MEM_CONTEXT_OLD()); + result.backupCopyResult = backupCopyResultNoOp; + result.copySize = pgTestSize; + result.copyChecksum = strDup(pgTestChecksum); + memContextSwitch(MEM_CONTEXT_TEMP()); + } + } + } + // Else the source file is missing from the database so skip this file + else + result.backupCopyResult = backupCopyResultSkip; + } + + // If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also test the + // checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, then there may + // be corruption in the repo, so recopy + if (!delta || !repoFileHasReference) + { + // If this is a delta backup and the file is missing from the DB, then remove it from the repo (backupManifestUpdate + // will remove it from the manifest) + if (result.backupCopyResult == backupCopyResultSkip) + { + storageRemoveNP(storageRepoWrite(), repoPathFile); + } + else if (!delta || pgFileMatch) + { + // Generate checksum/size for the repo file + IoRead *read = storageReadIo(storageNewReadNP(storageRepo(), repoPathFile)); + + if (cipherType != cipherTypeNone) + { + ioFilterGroupAdd( + ioReadFilterGroup(read), cipherBlockNew(cipherModeDecrypt, cipherType, BUFSTR(cipherPass), NULL)); + } + + if (repoFileCompress) + ioFilterGroupAdd(ioReadFilterGroup(read), gzipDecompressNew(false)); + + ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR)); + ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew()); + + ioReadDrain(read); + + // Test checksum/size + const String *pgTestChecksum = varStr( + ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR)); + uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR)); + + // No need to recopy if checksum/size match + if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum)) + { + memContextSwitch(MEM_CONTEXT_OLD()); + result.backupCopyResult = backupCopyResultChecksum; + result.copySize = pgTestSize; + result.copyChecksum = strDup(pgTestChecksum); + memContextSwitch(MEM_CONTEXT_TEMP()); + } + // Else recopy when repo file is not as expected + else + result.backupCopyResult = backupCopyResultReCopy; + } + } + } + + // Copy the file + if (result.backupCopyResult == backupCopyResultCopy || result.backupCopyResult == backupCopyResultReCopy) + { + // Is the file compressible during the copy? + bool compressible = !repoFileCompress && cipherType == cipherTypeNone; + + // Setup pg file for read + StorageRead *read = storageNewReadP( + storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR)); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew()); + + // Add page checksum filter + if (pgFileChecksumPage) + { + ioFilterGroupAdd( + ioReadFilterGroup(storageReadIo(read)), pageChecksumNew(segmentNumber(pgFile), PG_SEGMENT_PAGE_DEFAULT, + PG_PAGE_SIZE_DEFAULT, pgFileChecksumPageLsnLimit)); + } + + // Add compression + if (repoFileCompress) + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzipCompressNew((int)repoFileCompressLevel, false)); + + // If there is a cipher then add the encrypt filter + if (cipherType != cipherTypeNone) + { + ioFilterGroupAdd( + ioReadFilterGroup( + storageReadIo(read)), cipherBlockNew(cipherModeEncrypt, cipherType, BUFSTR(cipherPass), NULL)); + } + + // Setup the repo file for write + StorageWrite *write = storageNewWriteP(storageRepoWrite(), repoPathFile, .compressible = compressible); + ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), ioSizeNew()); + + // Open the source and destination and copy the file + if (storageCopy(read, write)) + { + memContextSwitch(MEM_CONTEXT_OLD()); + + // Get sizes and checksum + result.copySize = varUInt64Force( + ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE_STR)); + result.copyChecksum = strDup( + varStr(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR))); + result.repoSize = + varUInt64Force(ioFilterGroupResult(ioWriteFilterGroup(storageWriteIo(write)), SIZE_FILTER_TYPE_STR)); + + // Get results of page checksum validation + if (pgFileChecksumPage) + { + result.pageChecksumResult = kvDup( + varKv(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE_STR))); + } + + memContextSwitch(MEM_CONTEXT_TEMP()); + } + // Else if source file is missing and the read setup indicated ignore a missing file, the database removed it so skip it + else + result.backupCopyResult = backupCopyResultSkip; + } + + // If the file was copied get the repo size only if the storage can store the files with a different size than what was + // written. This has to be checked after the file is at rest because filesystem compression may affect the actual repo size + // and this cannot be calculated in stream. + // + // If the file was checksummed then get the size in all cases since we don't already have it. + if (((result.backupCopyResult == backupCopyResultCopy || result.backupCopyResult == backupCopyResultReCopy) && + storageFeature(storageRepo(), storageFeatureCompress)) || + result.backupCopyResult == backupCopyResultChecksum) + { + result.repoSize = storageInfoNP(storageRepo(), repoPathFile).size; + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(BACKUP_FILE_RESULT, result); +} diff --git a/src/command/backup/file.h b/src/command/backup/file.h new file mode 100644 index 000000000..c86799e2c --- /dev/null +++ b/src/command/backup/file.h @@ -0,0 +1,48 @@ +/*********************************************************************************************************************************** +Backup File +***********************************************************************************************************************************/ +#ifndef COMMAND_BACKUP_FILE_H +#define COMMAND_BACKUP_FILE_H + +#include "common/crypto/common.h" +#include "common/type/keyValue.h" + +/*********************************************************************************************************************************** +Backup file types +***********************************************************************************************************************************/ +typedef enum +{ + backupCopyResultChecksum, + backupCopyResultCopy, + backupCopyResultReCopy, + backupCopyResultSkip, + backupCopyResultNoOp, +} BackupCopyResult; + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +typedef struct BackupFileResult +{ + BackupCopyResult backupCopyResult; + uint64_t copySize; + String *copyChecksum; + uint64_t repoSize; + KeyValue *pageChecksumResult; +} BackupFileResult; + +BackupFileResult backupFile( + const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage, + uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress, + unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_LOG_BACKUP_FILE_RESULT_TYPE \ + BackupFileResult +#define FUNCTION_LOG_BACKUP_FILE_RESULT_FORMAT(value, buffer, bufferSize) \ + objToLog(&value, "BackupFileResult", buffer, bufferSize) + + +#endif diff --git a/src/command/backup/protocol.c b/src/command/backup/protocol.c new file mode 100644 index 000000000..1ac910f57 --- /dev/null +++ b/src/command/backup/protocol.c @@ -0,0 +1,67 @@ +/*********************************************************************************************************************************** +Backup Protocol Handler +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "command/backup/file.h" +#include "command/backup/protocol.h" +#include "common/debug.h" +#include "common/io/io.h" +#include "common/log.h" +#include "common/memContext.h" +#include "config/config.h" +#include "storage/helper.h" + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +STRING_EXTERN(PROTOCOL_COMMAND_BACKUP_FILE_STR, PROTOCOL_COMMAND_BACKUP_FILE); + +/*********************************************************************************************************************************** +Process protocol requests +***********************************************************************************************************************************/ +bool +backupProtocol(const String *command, const VariantList *paramList, ProtocolServer *server) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STRING, command); + FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); + FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); + FUNCTION_LOG_END(); + + ASSERT(command != NULL); + + // Attempt to satisfy the request -- we may get requests that are meant for other handlers + bool found = true; + + MEM_CONTEXT_TEMP_BEGIN() + { + if (strEq(command, PROTOCOL_COMMAND_BACKUP_FILE_STR)) + { + // Backup the file + BackupFileResult result = backupFile( + varStr(varLstGet(paramList, 0)), varBoolForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)), + varStr(varLstGet(paramList, 3)), varBoolForce(varLstGet(paramList, 4)), + varUInt64(varLstGet(paramList, 5)) << 32 | varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)), + varBoolForce(varLstGet(paramList, 8)), varBoolForce(varLstGet(paramList, 9)), + varUIntForce(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)), varBoolForce(varLstGet(paramList, 12)), + varLstSize(paramList) == 14 ? cipherTypeAes256Cbc : cipherTypeNone, + varLstSize(paramList) == 14 ? varStr(varLstGet(paramList, 13)) : NULL); + + // Return backup result + VariantList *resultList = varLstNew(); + varLstAdd(resultList, varNewUInt(result.backupCopyResult)); + varLstAdd(resultList, varNewUInt64(result.copySize)); + varLstAdd(resultList, varNewUInt64(result.repoSize)); + varLstAdd(resultList, varNewStr(result.copyChecksum)); + varLstAdd(resultList, result.pageChecksumResult != NULL ? varNewKv(result.pageChecksumResult) : NULL); + + protocolServerResponse(server, varNewVarLst(resultList)); + } + else + found = false; + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(BOOL, found); +} diff --git a/src/command/backup/protocol.h b/src/command/backup/protocol.h new file mode 100644 index 000000000..c7f1aefdc --- /dev/null +++ b/src/command/backup/protocol.h @@ -0,0 +1,22 @@ +/*********************************************************************************************************************************** +Backup Protocol Handler +***********************************************************************************************************************************/ +#ifndef COMMAND_BACKUP_PROTOCOL_H +#define COMMAND_BACKUP_PROTOCOL_H + +#include "common/type/string.h" +#include "common/type/variantList.h" +#include "protocol/server.h" + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +#define PROTOCOL_COMMAND_BACKUP_FILE "backupFile" + STRING_DECLARE(PROTOCOL_COMMAND_BACKUP_FILE_STR); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +bool backupProtocol(const String *command, const VariantList *paramList, ProtocolServer *server); + +#endif diff --git a/src/command/local/local.c b/src/command/local/local.c index e2a1e2263..0a613a6e7 100644 --- a/src/command/local/local.c +++ b/src/command/local/local.c @@ -5,6 +5,7 @@ Local Command #include "command/archive/get/protocol.h" #include "command/archive/push/protocol.h" +#include "command/backup/protocol.h" #include "command/restore/protocol.h" #include "common/debug.h" #include "common/io/handleRead.h" @@ -34,6 +35,7 @@ cmdLocal(int handleRead, int handleWrite) ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_LOCAL_STR, read, write); protocolServerHandlerAdd(server, archiveGetProtocol); protocolServerHandlerAdd(server, archivePushProtocol); + protocolServerHandlerAdd(server, backupProtocol); protocolServerHandlerAdd(server, restoreProtocol); protocolServerProcess(server); } diff --git a/src/main.c b/src/main.c index e847a6f8b..72ebf9a42 100644 --- a/src/main.c +++ b/src/main.c @@ -158,15 +158,7 @@ main(int argListSize, const char *argList[]) // ----------------------------------------------------------------------------------------------------------------- case cfgCmdLocal: { - if (strEq(cfgOptionStr(cfgOptCommand), CFGCMD_ARCHIVE_GET_ASYNC_STR) || - strEq(cfgOptionStr(cfgOptCommand), CFGCMD_ARCHIVE_PUSH_ASYNC_STR) || - strEq(cfgOptionStr(cfgOptCommand), CFGCMD_RESTORE_STR)) - { - cmdLocal(STDIN_FILENO, STDOUT_FILENO); - } - else - perlExec(); - + cmdLocal(STDIN_FILENO, STDOUT_FILENO); break; } diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index 3264f0c89..9cb35ae64 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -1165,12 +1165,6 @@ static const EmbeddedModule embeddedModule[] = "}\n" "}\n" "\n\n" - "my $hStartLsnParam =\n" - "{\n" - "iWalId => defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,\n" - "iWalOffset => defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,\n" - "};\n" - "\n\n" "foreach my $strRepoFile (\n" "sort {sprintf(\"%016d-%s\", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $b, MANIFEST_SUBKEY_SIZE), $b) cmp\n" "sprintf(\"%016d-%s\", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $a, MANIFEST_SUBKEY_SIZE), $a)}\n" @@ -1219,13 +1213,13 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "$oBackupProcess->queueJob(\n" "$iHostConfigIdx, $strQueueKey, $strRepoFile, OP_BACKUP_FILE,\n" - "[$strDbFile, $strRepoFile, $lSize,\n" + "[$strDbFile, $bIgnoreMissing, $lSize,\n" "$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, false),\n" - "cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $strBackupLabel, $bCompress,\n" - "cfgOption(CFGOPT_COMPRESS_LEVEL), $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile,\n" - "MANIFEST_SUBKEY_TIMESTAMP, false), $bIgnoreMissing,\n" - "cfgOption(CFGOPT_CHECKSUM_PAGE) && isChecksumPage($strRepoFile) ? $hStartLsnParam : undef,\n" - "cfgOption(CFGOPT_DELTA), defined($strReference) ? true : false],\n" + "cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false,\n" + "defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,\n" + "defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,\n" + "$strRepoFile, defined($strReference) ? true : false, $bCompress, cfgOption(CFGOPT_COMPRESS_LEVEL),\n" + "$strBackupLabel, cfgOption(CFGOPT_DELTA)],\n" "{rParamSecure => $oBackupManifest->cipherPassSub() ? [$oBackupManifest->cipherPassSub()] : undef});\n" "\n\n" "$oBackupManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_SIZE);\n" @@ -1260,7 +1254,8 @@ static const EmbeddedModule embeddedModule[] = "{\n" "($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(\n" "$oBackupManifest, cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $hJob->{iHostConfigIdx}), false),\n" - "$hJob->{iProcessId}, @{$hJob->{rParam}}[0..4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,\n" + "$hJob->{iProcessId}, @{$hJob->{rParam}}[0], @{$hJob->{rParam}}[7], @{$hJob->{rParam}}[2], @{$hJob->{rParam}}[3],\n" + "@{$hJob->{rParam}}[4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,\n" "$lManifestSaveCurrent);\n" "}\n" "\n\n\n" @@ -2092,204 +2087,6 @@ static const EmbeddedModule embeddedModule[] = "use constant BACKUP_FILE_NOOP => 4;\n" "push @EXPORT, qw(BACKUP_FILE_NOOP);\n" "\n\n\n\n" - "sub backupFile\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strDbFile,\n" - "$strRepoFile,\n" - "$lSizeFile,\n" - "$strChecksum,\n" - "$bChecksumPage,\n" - "$strBackupLabel,\n" - "$bCompress,\n" - "$iCompressLevel,\n" - "$lModificationTime,\n" - "$bIgnoreMissing,\n" - "$hExtraParam,\n" - "$bDelta,\n" - "$bHasReference,\n" - "$strCipherPass,\n" - "\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::backupFile', \\@_,\n" - "{name => 'strDbFile', trace => true},\n" - "{name => 'strRepoFile', trace => true},\n" - "{name => 'lSizeFile', trace => true},\n" - "{name => 'strChecksum', required => false, trace => true},\n" - "{name => 'bChecksumPage', trace => true},\n" - "{name => 'strBackupLabel', trace => true},\n" - "{name => 'bCompress', trace => true},\n" - "{name => 'iCompressLevel', trace => true},\n" - "{name => 'lModificationTime', trace => true},\n" - "{name => 'bIgnoreMissing', default => true, trace => true},\n" - "{name => 'hExtraParam', required => false, trace => true},\n" - "{name => 'bDelta', trace => true},\n" - "{name => 'bHasReference', trace => true},\n" - "{name => 'strCipherPass', required => false, trace => true},\n" - ");\n" - "\n" - "my $oStorageRepo = storageRepo();\n" - "my $iCopyResult = BACKUP_FILE_COPY;\n" - "my $strCopyChecksum;\n" - "my $rExtra;\n" - "my $lCopySize;\n" - "my $lRepoSize;\n" - "\n\n" - "my $strFileOp = $strRepoFile . ($bCompress ? '.' . COMPRESS_EXT : '');\n" - "\n" - "my $bCopy = true;\n" - "\n\n\n" - "if (defined($strChecksum))\n" - "{\n" - "\n" - "if ($bDelta)\n" - "{\n" - "($strCopyChecksum, $lCopySize) = storageDb()->hashSize($strDbFile, {bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "if (defined($strCopyChecksum))\n" - "{\n" - "$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);\n" - "\n\n\n\n" - "if (!$bCopy && $bHasReference)\n" - "{\n" - "$iCopyResult = BACKUP_FILE_NOOP;\n" - "}\n" - "}\n" - "\n" - "else\n" - "{\n" - "$iCopyResult = BACKUP_FILE_SKIP;\n" - "$bCopy = false;\n" - "}\n" - "}\n" - "\n\n\n\n" - "if (!$bDelta || !$bHasReference)\n" - "{\n" - "\n\n" - "if ($iCopyResult == BACKUP_FILE_SKIP)\n" - "{\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\");\n" - "}\n" - "elsif (!$bDelta || !$bCopy)\n" - "{\n" - "\n" - "my $rhyFilter;\n" - "\n" - "if ($bCompress)\n" - "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]});\n" - "}\n" - "\n\n" - "($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize(\n" - "$oStorageRepo->openRead(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\",\n" - "{rhyFilter => $rhyFilter, strCipherPass => $strCipherPass}));\n" - "\n\n" - "$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);\n" - "\n\n" - "$iCopyResult = $bCopy ? BACKUP_FILE_RECOPY : BACKUP_FILE_CHECKSUM;\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "if ($bCopy)\n" - "{\n" - "\n" - "my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}];\n" - "\n\n" - "if ($bChecksumPage)\n" - "{\n" - "\n" - "my $iSegmentNo = ($strDbFile =~ /\\.[0-9]+$/) ? substr(($strDbFile =~ m/\\.[0-9]+$/g)[0], 1) + 0 : 0;\n" - "\n" - "push(\n" - "@{$rhyFilter},\n" - "{strClass => \"pgBackRest::Backup::Filter::PageChecksum\",\n" - "rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});\n" - "};\n" - "\n\n" - "if ($bCompress)\n" - "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]});\n" - "}\n" - "\n" - "elsif (!defined($strCipherPass))\n" - "{\n" - "push(\n" - "@{$rhyFilter},\n" - "{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]});\n" - "}\n" - "\n\n" - "my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "$rhyFilter = undef;\n" - "\n\n" - "if (!$bCompress && !defined($strCipherPass))\n" - "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]});\n" - "}\n" - "\n" - "my $oDestinationFileIo = $oStorageRepo->openWrite(\n" - "STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\",\n" - "{bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass});\n" - "\n" - "$oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef);\n" - "\n\n" - "if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo))\n" - "{\n" - "\n" - "$strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);\n" - "$lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);\n" - "$lRepoSize = $oDestinationFileIo->result(COMMON_IO_HANDLE);\n" - "\n" - "if (!defined($lRepoSize))\n" - "{\n" - "confess &log(ERROR, \"REPO_SIZE IS NOT SET\");\n" - "}\n" - "\n\n" - "if ($bChecksumPage)\n" - "{\n" - "my $rExtraRaw = $oSourceFileIo->result(\"pgBackRest::Backup::Filter::PageChecksum\");\n" - "\n" - "$rExtra =\n" - "{\n" - "bValid => $rExtraRaw->{valid} ? true : false,\n" - "bAlign => $rExtraRaw->{align} ? true : false,\n" - "iyPageError => $rExtraRaw->{error},\n" - "};\n" - "}\n" - "}\n" - "\n" - "else\n" - "{\n" - "$iCopyResult = BACKUP_FILE_SKIP;\n" - "}\n" - "}\n" - "\n\n\n\n\n\n" - "if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) &&\n" - "$oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||\n" - "$iCopyResult == BACKUP_FILE_CHECKSUM)\n" - "{\n" - "$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\"))->{size};\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'iCopyResult', value => $iCopyResult},\n" - "{name => 'lCopySize', value => $lCopySize},\n" - "{name => 'lRepoSize', value => $lRepoSize},\n" - "{name => 'strCopyChecksum', value => $strCopyChecksum},\n" - "{name => 'rExtra', value => $rExtra},\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(backupFile);\n" - "\n\n\n\n" "sub backupManifestUpdate\n" "{\n" "\n" @@ -2397,20 +2194,21 @@ static const EmbeddedModule embeddedModule[] = "if ($bChecksumPage)\n" "{\n" "\n" - "if (defined($rExtra->{bValid}))\n" + "if (defined($rExtra->{valid}))\n" "{\n" "\n" - "$oManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{bValid});\n" + "$oManifest->boolSet(\n" + "MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{valid});\n" "\n\n" - "if (!$rExtra->{bValid})\n" + "if (!$rExtra->{valid})\n" "{\n" "\n" "if ($lSizeCopy % PG_PAGE_SIZE != 0)\n" "{\n" "\n" - "if (!defined($rExtra->{bAlign}) || $rExtra->{bAlign})\n" + "if (!defined($rExtra->{align}) || $rExtra->{align})\n" "{\n" - "confess &log(ASSERT, 'bAlign flag should have been set for misaligned page');\n" + "confess &log(ASSERT, 'align flag should have been set for misaligned page');\n" "}\n" "\n\n" "&log(WARN,\n" @@ -2422,12 +2220,12 @@ static const EmbeddedModule embeddedModule[] = "{\n" "$oManifest->set(\n" "MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,\n" - "dclone($rExtra->{iyPageError}));\n" + "dclone($rExtra->{error}));\n" "\n\n" "my $strPageError;\n" "my $iPageErrorTotal = 0;\n" "\n" - "foreach my $iyPage (@{$rExtra->{iyPageError}})\n" + "foreach my $iyPage (@{$rExtra->{error}})\n" "{\n" "$strPageError .= (defined($strPageError) ? ', ' : '');\n" "\n\n" @@ -8681,25 +8479,6 @@ static const EmbeddedModule embeddedModule[] = "cfgOption(CFGOPT_LOCK_PATH), cfgOption(CFGOPT_COMMAND), cfgOption(CFGOPT_STANZA, false), cfgOption(CFGOPT_PROCESS));\n" "}\n" "\n\n\n" - "elsif (cfgCommandTest(CFGCMD_LOCAL))\n" - "{\n" - "\n" - "cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);\n" - "logLevelSet(cfgOption(CFGOPT_LOG_LEVEL_FILE), OFF, cfgOption(CFGOPT_LOG_LEVEL_STDERR));\n" - "\n" - "logFileSet(\n" - "storageLocal(),\n" - "cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgOption(CFGOPT_COMMAND)) . '-' .\n" - "lc(cfgCommandName(cfgCommandGet())) . '-' . sprintf(\"%03d\", cfgOption(CFGOPT_PROCESS)));\n" - "\n\n" - "require pgBackRest::Protocol::Local::Minion;\n" - "pgBackRest::Protocol::Local::Minion->import();\n" - "\n\n" - "my $oLocal = new pgBackRest::Protocol::Local::Minion();\n" - "\n\n" - "$oLocal->process();\n" - "}\n" - "\n\n\n" "elsif (cfgCommandTest(CFGCMD_CHECK))\n" "{\n" "\n" @@ -11260,64 +11039,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Protocol/Local/Minion.pm", - .data = - "\n\n\n" - "package pgBackRest::Protocol::Local::Minion;\n" - "use parent 'pgBackRest::Protocol::Command::Minion';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "\n" - "use pgBackRest::Backup::File;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Config::Config;\n" - "use pgBackRest::Protocol::Base::Master;\n" - "use pgBackRest::Protocol::Base::Minion;\n" - "use pgBackRest::Protocol::Command::Minion;\n" - "use pgBackRest::Protocol::Helper;\n" - "use pgBackRest::RestoreFile;\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');\n" - "\n\n" - "my $self = $class->SUPER::new(cfgCommandName(CFGCMD_LOCAL), cfgOption(CFGOPT_BUFFER_SIZE), cfgOption(CFGOPT_PROTOCOL_TIMEOUT));\n" - "bless $self, $class;\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub init\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');\n" - "\n\n" - "my $hCommandMap =\n" - "{\n" - "&OP_BACKUP_FILE => sub {backupFile(@{shift()})},\n" - "\n\n" - "&OP_POST => sub {protocolKeepAlive()},\n" - "};\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hCommandMap', value => $hCommandMap}\n" - ");\n" - "}\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/Protocol/Local/Process.pm", .data = diff --git a/test/define.yaml b/test/define.yaml index 6ec56640a..b0e2a6d41 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -573,6 +573,15 @@ unit: command/backup/common: full command/backup/pageChecksum: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: backup + total: 3 + + coverage: + command/backup/file: full + command/backup/protocol: full + storage/storage: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: command total: 1 diff --git a/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm b/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm index 9c6015d5d..4837e5ff8 100644 --- a/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm @@ -125,16 +125,8 @@ sub run if ($self->begin('backupFile(), backupManifestUpdate()')) { #--------------------------------------------------------------------------------------------------------------------------- - # Copy pg_control and confirm manifestUpdate does not save the manifest yet - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($self->{strPgControl}, MANIFEST_FILE_PGCONTROL, $lPgControlSize, undef, false, $strBackupLabel, false, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lPgControlTime, true, undef, false, false, undef); - - $self->testResult(sub {storageTest()->exists($strPgControlRepo)}, true, 'pg_control file exists in repo'); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strPgControlHash && - $lResultCopySize == $lPgControlSize && $lResultRepoSize == $lPgControlSize), true, - 'pg_control file copied to repo successfully'); + # Create backup path so manifest can be saved + storageRepo->pathCreate(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/$strBackupLabel")); ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, @@ -145,15 +137,15 @@ sub run $lPgControlSize, undef, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + BACKUP_FILE_COPY, + 8192, + 8192, + $strPgControlHash, + undef, + 16785408, + 0, + 167854, + 0); # Accumulators should be same size as pg_control $self->testResult(($lSizeCurrent == $lPgControlSize && $lManifestSaveCurrent == $lPgControlSize), true, @@ -169,18 +161,6 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- # No prior checksum, no compression, no page checksum, no extra, no delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, undef, false, $strBackupLabel, false, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef); - - $self->testResult(sub {storageTest()->exists($strFileRepo)}, true, 'non-compressed file exists in repo'); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, - 'file copied to repo successfully'); - - $self->testResult(sub {storageRepo()->exists("${strFileRepo}.gz")}, false, "${strFileRepo}.gz missing"); - ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, $strHost, @@ -190,15 +170,15 @@ sub run $lFileSize, $strFileHash, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + BACKUP_FILE_COPY, + 16777216, + 16777216, + '1c7e00fd09b9dd11fc2966590b3e3274645dd031', + undef, + 16785408, + 8192, + 167854, + 8192); # Accumulator includes size of pg_control and file. Manifest saved so ManifestSaveCurrent returns to 0 $self->testResult(($lSizeCurrent == ($lPgControlSize + $lFileSize) && $lManifestSaveCurrent == 0), true, @@ -213,29 +193,9 @@ sub run $self->testResult( sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, 'backup.manifest.copy missing in repo'); - storageTest()->remove($strFileRepo); - storageTest()->remove($strPgControlRepo); - #--------------------------------------------------------------------------------------------------------------------------- - # Build the lsn start parameter to pass to the extra function - my $hStartLsnParam = - { - iWalId => 0xFFFF, - iWalOffset => 0xFFFF, - }; - - # No prior checksum, yes compression, yes page checksum, yes extra, no delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true, $strBackupLabel, true, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, $hStartLsnParam, false, false, undef); - - $self->testResult(sub {storageTest()->exists("$strFileRepo.gz")}, true, 'compressed file exists in repo'); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultRepoSize == $lRepoFileCompressSize && $rResultExtra->{bValid}), true, 'file copied to repo successfully'); - - # Only the compressed version of the file exists - $self->testResult(sub {storageRepo()->exists("$strFileRepo")}, false, "only compressed version exists"); + # Set up page checksum result + $rResultExtra = {'valid' => true,'align' => true}; ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, @@ -246,15 +206,15 @@ sub run $lFileSize, $strFileHash, true, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, + BACKUP_FILE_COPY, + 16777216, + 3646899, + '1c7e00fd09b9dd11fc2966590b3e3274645dd031', $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + 16785408, + 16785408, + 167854, + 0); # File is compressed in repo so make sure repo-size added to manifest $self->testResult(sub {$oBackupManifest->test( @@ -264,45 +224,15 @@ sub run MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rResultExtra->{bValid})}, true, "checksum page set"); - # Save the compressed file for later test - executeTest('mv ' . "$strFileRepo.gz $strFileRepo.gz.SAVE"); - - #--------------------------------------------------------------------------------------------------------------------------- - # Add a segment number for bChecksumPage code coverage - executeTest('cp ' . "$strFileDb $strFileDb.1"); - - # No prior checksum, no compression, yes page checksum, yes extra, no delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, undef, true, $strBackupLabel, false, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, $hStartLsnParam, false, false, undef); - - $self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, true, 'non-compressed segment file exists in repo'); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultRepoSize == $lFileSize && $rResultExtra->{bValid}), true, 'segment file copied to repo successfully'); - # Set a section in the manifest to ensure it is removed in the next test $oBackupManifest->set( MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1", MANIFEST_SUBKEY_CHECKSUM, $strResultCopyChecksum); $self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.1")}, - true, MANIFEST_TARGET_PGDATA . "/$strFileName.1 section exists in manifest"); + true, MANIFEST_TARGET_PGDATA . "/$strFileName.1 section exists in manifest - skip file"); #--------------------------------------------------------------------------------------------------------------------------- - # Remove the db file and try to back it up - storageTest()->remove("$strFileDb.1"); - - # No prior checksum, no compression, no page checksum, no extra, No delta, no hasReference, no db file - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, undef, false, $strBackupLabel, - false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_SKIP && !defined($strResultCopyChecksum) && - !defined($lResultRepoSize) && !defined($lResultCopySize)), true, "db file missing - $strRepoFile.1 file skipped"); - - # Delta not set so file still exists in repo - $self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, true, ' delta not set - file exists in repo'); - + # Removed db file is removed from manifest ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, $strHost, @@ -312,30 +242,23 @@ sub run $lFileSize, $strFileHash, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + BACKUP_FILE_SKIP, + undef, + undef, + undef, + undef, + 16785408, + 33562624, + 167854, + 0); $self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")}, false, " $strRepoFile.1 section removed from manifest"); - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference, no db file - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile("$strFileDb.1", MANIFEST_TARGET_PGDATA . "/$strFileName.1", $lFileSize, $strFileHash, false, $strBackupLabel, - false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef); + # Add back the section + $oBackupManifest->set(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1"); - $self->testResult(($iResultCopyResult == BACKUP_FILE_SKIP && !defined($strResultCopyChecksum) && - !defined($lResultRepoSize)), true, "db file missing - delta $strRepoFile.1 file skipped"); - - $self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, false, ' delta set - file removed from repo'); - - # Code path for host not defined for logged message of skipped file + # Code coverage for code path when host not defined for logged message of skipped file ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, undef, @@ -345,100 +268,21 @@ sub run $lFileSize, $strFileHash, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + BACKUP_FILE_SKIP, + undef, + undef, + undef, + undef, + 16785408, + 50339840, + 167854, + 0); - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference, no db file, - # do not ignoreMissing - $self->testException(sub {backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, $strFileHash, - false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, false, undef, true, false, undef)}, - ERROR_FILE_MISSING, "unable to open missing file '${strFileDb}.1' for read"); + $self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")}, + false, " $strRepoFile.1 section removed from manifest on undef host"); #--------------------------------------------------------------------------------------------------------------------------- - # Restore the compressed file - executeTest('mv ' . "$strFileRepo.gz.SAVE $strFileRepo.gz"); - - # Yes prior checksum, yes compression, no page checksum, no extra, yes delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, $strBackupLabel, - true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_CHECKSUM && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize), true, 'db checksum and repo same - no copy file'); - - #--------------------------------------------------------------------------------------------------------------------------- - # DB Checksum mismatch - storageTest()->remove("$strFileRepo", {bIgnoreMissing => true}); - # Save the compressed file for later test - executeTest('mv ' . "$strFileRepo.gz $strFileRepo.gz.SAVE"); - - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash . "ff", false, - $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db checksum mismatch - copy file'); - - #--------------------------------------------------------------------------------------------------------------------------- - # DB file size mismatch - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false, $strBackupLabel, false, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db file size mismatch - copy file'); - - #--------------------------------------------------------------------------------------------------------------------------- - # Repo mismatch - - # Restore the compressed file as if non-compressed so checksum won't match - executeTest('cp ' . "$strFileRepo.gz.SAVE $strFileRepo"); - - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, $strBackupLabel, false, - cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_RECOPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'repo checksum mismatch - recopy file'); - - # Restore the compressed file - executeTest('mv ' . "$strFileRepo.gz.SAVE $strFileRepo.gz"); - - # Yes prior checksum, yes compression, no page checksum, no extra, no delta, no hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false, - $strBackupLabel, true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_RECOPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize), true, 'repo size mismatch - recopy file'); - - #--------------------------------------------------------------------------------------------------------------------------- - # Has reference - # Set a reference in the manifest to ensure it is removed after backupManifestUpdate - $oBackupManifest->set(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REFERENCE, BOGUS); - - $self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REFERENCE, - BOGUS)}, true, "$strRepoFile reference section exists in manifest"); - - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, yes hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false, - $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, true, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db file size mismatch has reference - copy'); - - # Code path to ensure reference is removed + # Has reference - Code path to ensure reference is removed ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, $strHost, @@ -448,15 +292,15 @@ sub run $lFileSize, $strFileHash, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, - $lManifestSaveSize, - $lManifestSaveCurrent); + BACKUP_FILE_COPY, + 16777216, + 16777216, + '1c7e00fd09b9dd11fc2966590b3e3274645dd031', + undef, + 16785408, + 67117056, + 167854, + 0); # Confirm reference to prior backup removed $self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.", @@ -465,13 +309,6 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- # BACKUP_FILE_NOOP - # Yes prior checksum, no compression, no page checksum, no extra, yes delta, yes hasReference - ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = - backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, - $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, true, undef); - - $self->testResult(($iResultCopyResult == BACKUP_FILE_NOOP && $strResultCopyChecksum eq $strFileHash && - $lResultCopySize == $lFileSize), true, 'db file same has reference - noop'); # Calculate running counts my $lSizeCurrentAfter = $lSizeCurrent + $lFileSize; @@ -489,27 +326,18 @@ sub run $lFileSize, $strFileHash, false, - $iResultCopyResult, - $lResultCopySize, - $lResultRepoSize, - $strResultCopyChecksum, - $rResultExtra, - $lSizeTotal, - $lSizeCurrent, + BACKUP_FILE_NOOP, + 16777216, + undef, + '1c7e00fd09b9dd11fc2966590b3e3274645dd031', + undef, + 16785408, + 83894272, $lManifestSaveSize, - $lManifestSaveCurrent); + 0); $self->testResult(($lSizeCurrent ==$lSizeCurrentAfter && $lManifestSaveCurrent == $lManifestSaveCurrentAfter), true, ' running counts updated'); - - #--------------------------------------------------------------------------------------------------------------------------- - # Remove file from repo. No reference so should hard error since this means sometime between the building of the manifest - # for the aborted backup, the file went missing from the aborted backup dir. - storageTest()->remove("$strFileRepo", {bIgnoreMissing => true}); - - $self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, - false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef)}, - ERROR_FILE_MISSING, "unable to open missing file '${strFileRepo}' for read"); } ################################################################################################################################ @@ -528,6 +356,7 @@ sub run $lSizeCurrent = 0; $lManifestSaveSize = $lFileSize * 2; $lManifestSaveCurrent = 0; + $rResultExtra = undef; $self->testResult(sub {backupManifestUpdate( $oBackupManifest, @@ -588,7 +417,7 @@ sub run $lManifestSaveCurrent)}, ERROR_ASSERT, "$strFileDb should have calculated page checksums"); - $rResultExtra->{bValid} = false; + $rResultExtra->{valid} = false; $self->testException(sub {backupManifestUpdate( $oBackupManifest, $strHost, @@ -607,9 +436,9 @@ sub run $lSizeCurrent, $lManifestSaveSize, $lManifestSaveCurrent)}, - ERROR_ASSERT, "bAlign flag should have been set for misaligned page"); + ERROR_ASSERT, "align flag should have been set for misaligned page"); - $rResultExtra->{bAlign} = true; + $rResultExtra->{align} = true; $self->testException(sub {backupManifestUpdate( $oBackupManifest, $strHost, @@ -628,9 +457,9 @@ sub run $lSizeCurrent, $lManifestSaveSize, $lManifestSaveCurrent)}, - ERROR_ASSERT, "bAlign flag should have been set for misaligned page"); + ERROR_ASSERT, "align flag should have been set for misaligned page"); - $rResultExtra->{bAlign} = false; + $rResultExtra->{align} = false; $self->testResult(sub {backupManifestUpdate( $oBackupManifest, $strHost, diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c new file mode 100644 index 000000000..284858bc2 --- /dev/null +++ b/test/src/module/command/backupTest.c @@ -0,0 +1,457 @@ +/*********************************************************************************************************************************** +Test Backup Command +***********************************************************************************************************************************/ +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" +#include "common/io/io.h" +#include "storage/helper.h" +#include "storage/posix/storage.h" + +#include "common/harnessConfig.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // Start a protocol server to test the protocol directly + Buffer *serverWrite = bufNew(8192); + IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite); + ioWriteOpen(serverWriteIo); + + ProtocolServer *server = protocolServerNew(strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo); + bufUsedSet(serverWrite, 0); + + const String *pgFile = strNew("testfile"); + const String *missingFile = strNew("missing"); + const String *backupLabel = strNew("20190718-155825F"); + const String *backupPathFile = strNewFmt(STORAGE_REPO_BACKUP "/%s/%s", strPtr(backupLabel), strPtr(pgFile)); + BackupFileResult result = {0}; + VariantList *paramList = varLstNew(); + + // ***************************************************************************************************************************** + if (testBegin("segmentNumber()")) + { + TEST_RESULT_UINT(segmentNumber(pgFile), 0, "No segment number"); + TEST_RESULT_UINT(segmentNumber(strNewFmt("%s.123", strPtr(pgFile))), 123, "Segment number"); + } + + // ***************************************************************************************************************************** + if (testBegin("backupFile(), backupProtocol")) + { + // Load Parameters + StringList *argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "--stanza=test1"); + strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); + strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath())); + strLstAddZ(argList, "--repo1-retention-full=1"); + strLstAddZ(argList, "backup"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + // Create the pg path + storagePathCreateP(storagePgWrite(), NULL, .mode = 0700); + + // Pg file missing - ignoreMissing=true + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN( + result, + backupFile( + missingFile, true, 0, NULL, false, 0, missingFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL), + "pg file missing, ignoreMissing=true, no delta"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy/repo size 0"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, " skip file"); + + // Check protocol function directly + // ------------------------------------------------------------------------------------------------------------------------- + // NULL, zero param values, ignoreMissing=true + varLstAdd(paramList, varNewStr(missingFile)); // pgFile + varLstAdd(paramList, varNewBool(true)); // pgFileIgnoreMissing + varLstAdd(paramList, varNewUInt64(0)); // pgFileSize + varLstAdd(paramList, NULL); // pgFileChecksum + varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1 + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2 + varLstAdd(paramList, varNewStr(missingFile)); // repoFile + varLstAdd(paramList, varNewBool(false)); // repoFileHasReference + varLstAdd(paramList, varNewBool(false)); // repoFileCompress + varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel + varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel + varLstAdd(paramList, varNewBool(false)); // delta + + TEST_RESULT_BOOL( + backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - skip"); + TEST_RESULT_STR(strPtr(strNewBuf(serverWrite)), "{\"out\":[3,0,0,null,null]}\n", " check result"); + bufUsedSet(serverWrite, 0); + + // Pg file missing - ignoreMissing=false + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ERROR_FMT( + backupFile( + missingFile, false, 0, NULL, false, 0, missingFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL), + FileMissingError, "unable to open missing file '%s/pg/missing' for read", testPath()); + + // Create a pg file to backup + storagePutNP(storageNewWriteNP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile")); + + // ------------------------------------------------------------------------------------------------------------------------- + // No prior checksum, no compression, no pageChecksum, no delta, no hasReference + + // With the expected backupCopyResultCopy, unset the storageFeatureCompress bit for the storageRepo for code coverage + uint64_t feature = storageRepo()->interface.feature; + ((Storage *)storageRepo())->interface.feature = feature && ((1 << storageFeatureCompress) ^ 0xFFFFFFFFFFFFFFFF); + + TEST_ASSIGN( + result, + backupFile(pgFile, false, 9, NULL, false, 0, pgFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL), + "pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference"); + + ((Storage *)storageRepo())->interface.feature = feature; + + TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " copy file to repo success"); + + TEST_RESULT_VOID(storageRemoveNP(storageRepoWrite(), backupPathFile), " remove repo file"); + + // ------------------------------------------------------------------------------------------------------------------------- + // Test pagechecksum + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, NULL, true, 0xFFFFFFFFFFFFFFFF, pgFile, false, false, 1, backupLabel, false, cipherTypeNone, + NULL), + "file checksummed with pageChecksum enabled"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile)), + true," copy file to repo success"); + TEST_RESULT_PTR_NE(result.pageChecksumResult, NULL, " pageChecksumResult is set"); + TEST_RESULT_BOOL( + varBool(kvGet(result.pageChecksumResult, VARSTRDEF("valid"))), false, " pageChecksumResult valid=false"); + TEST_RESULT_VOID(storageRemoveNP(storageRepoWrite(), backupPathFile), " remove repo file"); + + // Check protocol function directly + // ------------------------------------------------------------------------------------------------------------------------- + // pgFileSize, ignoreMissing=false, backupLabel, pgFileChecksumPage, pgFileChecksumPageLsnLimit + paramList = varLstNew(); + varLstAdd(paramList, varNewStr(pgFile)); // pgFile + varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing + varLstAdd(paramList, varNewUInt64(9)); // pgFileSize + varLstAdd(paramList, NULL); // pgFileChecksum + varLstAdd(paramList, varNewBool(true)); // pgFileChecksumPage + varLstAdd(paramList, varNewUInt64(0xFFFFFFFF)); // pgFileChecksumPageLsnLimit 1 + varLstAdd(paramList, varNewUInt64(0xFFFFFFFF)); // pgFileChecksumPageLsnLimit 2 + varLstAdd(paramList, varNewStr(pgFile)); // repoFile + varLstAdd(paramList, varNewBool(false)); // repoFileHasReference + varLstAdd(paramList, varNewBool(false)); // repoFileCompress + varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel + varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel + varLstAdd(paramList, varNewBool(false)); // delta + + TEST_RESULT_BOOL( + backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - pageChecksum"); + TEST_RESULT_STR( + strPtr(strNewBuf(serverWrite)), + "{\"out\":[1,9,9,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",{\"align\":false,\"valid\":false}]}\n", + " check result"); + bufUsedSet(serverWrite, 0); + + // ------------------------------------------------------------------------------------------------------------------------- + // File exists in repo and db, checksum match, delta set, ignoreMissing false, hasReference - NOOP + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel, + true, cipherTypeNone, NULL), + "file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference"); + TEST_RESULT_UINT(result.copySize, 9, " copy size set"); + TEST_RESULT_UINT(result.repoSize, 0, " repo size not set since already exists in repo"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultNoOp, " noop file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " noop"); + + // Check protocol function directly + // ------------------------------------------------------------------------------------------------------------------------- + // pgFileChecksum, hasReference, delta + paramList = varLstNew(); + varLstAdd(paramList, varNewStr(pgFile)); // pgFile + varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing + varLstAdd(paramList, varNewUInt64(9)); // pgFileSize + varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); // pgFileChecksum + varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1 + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2 + varLstAdd(paramList, varNewStr(pgFile)); // repoFile + varLstAdd(paramList, varNewBool(true)); // repoFileHasReference + varLstAdd(paramList, varNewBool(false)); // repoFileCompress + varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel + varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel + varLstAdd(paramList, varNewBool(true)); // delta + + TEST_RESULT_BOOL( + backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - noop"); + TEST_RESULT_STR( + strPtr(strNewBuf(serverWrite)), "{\"out\":[4,9,0,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n", + " check result"); + bufUsedSet(serverWrite, 0); + + // ------------------------------------------------------------------------------------------------------------------------- + // File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, strNew("1234567890123456789012345678901234567890"), false, 0, pgFile, true, false, 1, backupLabel, + true, cipherTypeNone, NULL), + "file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " copy"); + + // ------------------------------------------------------------------------------------------------------------------------- + // File exists in repo and db, pg checksum same, pg size different, delta set, ignoreMissing false, hasReference - COPY + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel, + true, cipherTypeNone, NULL), + "db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " copy"); + + // ------------------------------------------------------------------------------------------------------------------------- + // File exists in repo and db, checksum not same in repo, delta set, ignoreMissing false, no hasReference - RECOPY + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageRepoWrite(), backupPathFile), BUFSTRDEF("adifferentfile")), + "create different file (size and checksum) with same name in repo"); + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1, + backupLabel, true, cipherTypeNone, NULL), + " db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, " recopy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " recopy"); + + // ------------------------------------------------------------------------------------------------------------------------- + // File exists in repo but missing from db, checksum same in repo, delta set, ignoreMissing true, no hasReference - SKIP + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageRepoWrite(), backupPathFile), BUFSTRDEF("adifferentfile")), + "create different file with same name in repo"); + TEST_ASSIGN( + result, + backupFile( + missingFile, true, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1, + backupLabel, true, cipherTypeNone, NULL), + " file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy=repo=0 size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, " skip file"); + TEST_RESULT_BOOL( + (result.copyChecksum == NULL && !storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " skip and remove file from repo"); + + // ------------------------------------------------------------------------------------------------------------------------- + // No prior checksum, compression, no page checksum, no pageChecksum, no delta, no hasReference + TEST_ASSIGN( + result, + backupFile(pgFile, false, 9, NULL, false, 0, pgFile, false, true, 3, backupLabel, false, cipherTypeNone, NULL), + "pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference"); + + TEST_RESULT_UINT(result.copySize, 9, " copy=pgFile size"); + TEST_RESULT_UINT(result.repoSize, 29, " repo compress size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(backupLabel), strPtr(pgFile))) && + result.pageChecksumResult == NULL), + true, " copy file to repo compress success"); + + // ------------------------------------------------------------------------------------------------------------------------- + // Pg and repo file exist & match, prior checksum, compression, no page checksum, no pageChecksum, no delta, no hasReference + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, true, 3, backupLabel, + false, cipherTypeNone, NULL), + "pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference"); + + TEST_RESULT_UINT(result.copySize, 9, " copy=pgFile size"); + TEST_RESULT_UINT(result.repoSize, 29, " repo compress size"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultChecksum, " checksum file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(backupLabel), strPtr(pgFile))) && + result.pageChecksumResult == NULL), + true, " compressed repo file matches"); + + // Check protocol function directly + // ------------------------------------------------------------------------------------------------------------------------- + // compression + paramList = varLstNew(); + varLstAdd(paramList, varNewStr(pgFile)); // pgFile + varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing + varLstAdd(paramList, varNewUInt64(9)); // pgFileSize + varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); // pgFileChecksum + varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1 + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2 + varLstAdd(paramList, varNewStr(pgFile)); // repoFile + varLstAdd(paramList, varNewBool(false)); // repoFileHasReference + varLstAdd(paramList, varNewBool(true)); // repoFileCompress + varLstAdd(paramList, varNewUInt(3)); // repoFileCompressLevel + varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel + varLstAdd(paramList, varNewBool(false)); // delta + + TEST_RESULT_BOOL( + backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - copy, compress"); + TEST_RESULT_STR( + strPtr(strNewBuf(serverWrite)), "{\"out\":[0,9,29,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n", + " check result"); + bufUsedSet(serverWrite, 0); + + // ------------------------------------------------------------------------------------------------------------------------- + // Create a zero sized file - checksum will be set but in backupManifestUpdate it will not be copied + storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("zerofile")), BUFSTRDEF("")); + + // No prior checksum, no compression, no pageChecksum, no delta, no hasReference + TEST_ASSIGN( + result, + backupFile( + strNew("zerofile"), false, 0, NULL, false, 0, strNew("zerofile"), false, false, 1, backupLabel, false, + cipherTypeNone, NULL), + "zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference"); + TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy=repo=pgFile size 0"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_PTR_NE(result.copyChecksum, NULL, " checksum set"); + TEST_RESULT_BOOL( + (storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/zerofile", strPtr(backupLabel))) && + result.pageChecksumResult == NULL), + true, " copy zero file to repo success"); + + // Check invalid protocol function + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_BOOL(backupProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function"); + } + + // ***************************************************************************************************************************** + if (testBegin("backupFile() - encrypt")) + { + // Load Parameters + StringList *argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "--stanza=test1"); + strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); + strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath())); + strLstAddZ(argList, "--repo1-retention-full=1"); + strLstAddZ(argList, "--repo1-cipher-type=aes-256-cbc"); + strLstAddZ(argList, "backup"); + setenv("PGBACKREST_REPO1_CIPHER_PASS", "12345678", true); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + unsetenv("PGBACKREST_REPO1_CIPHER_PASS"); + + // Create the pg path + storagePathCreateP(storagePgWrite(), NULL, .mode = 0700); + + // Create a pg file to backup + storagePutNP(storageNewWriteNP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile")); + + // ------------------------------------------------------------------------------------------------------------------------- + // No prior checksum, no compression, no pageChecksum, no delta, no hasReference + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, NULL, false, 0, pgFile, false, false, 1, backupLabel, false, cipherTypeAes256Cbc, + strNew("12345678")), + "pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference"); + + TEST_RESULT_UINT(result.copySize, 9, " copy size set"); + TEST_RESULT_UINT(result.repoSize, 32, " repo size set"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " copy file to encrypted repo success"); + + // ------------------------------------------------------------------------------------------------------------------------- + // Delta but pgMatch false (pg File size different), prior checksum, no compression, no pageChecksum, delta, no hasReference + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1, + backupLabel, true, cipherTypeAes256Cbc, strNew("12345678")), + "pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference"); + TEST_RESULT_UINT(result.copySize, 9, " copy size set"); + TEST_RESULT_UINT(result.repoSize, 32, " repo size set"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " copy file (size missmatch) to encrypted repo success"); + + // ------------------------------------------------------------------------------------------------------------------------- + // Check repo with cipher filter. + // pg/repo file size same but checksum different, prior checksum, no compression, no pageChecksum, no delta, no hasReference + TEST_ASSIGN( + result, + backupFile( + pgFile, false, 9, strNew("1234567890123456789012345678901234567890"), false, 0, pgFile, false, false, 0, + backupLabel, false, cipherTypeAes256Cbc, strNew("12345678")), + "pg and repo file exists, repo checksum no match, no ignoreMissing, no pageChecksum, no delta, no hasReference"); + TEST_RESULT_UINT(result.copySize, 9, " copy size set"); + TEST_RESULT_UINT(result.repoSize, 32, " repo size set"); + TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, " recopy file"); + TEST_RESULT_BOOL( + (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") && + storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL), + true, " recopy file to encrypted repo success"); + + // Check protocol function directly + // ------------------------------------------------------------------------------------------------------------------------- + // cipherType, cipherPass + paramList = varLstNew(); + varLstAdd(paramList, varNewStr(pgFile)); // pgFile + varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing + varLstAdd(paramList, varNewUInt64(9)); // pgFileSize + varLstAdd(paramList, varNewStrZ("1234567890123456789012345678901234567890")); // pgFileChecksum + varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1 + varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2 + varLstAdd(paramList, varNewStr(pgFile)); // repoFile + varLstAdd(paramList, varNewBool(false)); // repoFileHasReference + varLstAdd(paramList, varNewBool(false)); // repoFileCompress + varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel + varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel + varLstAdd(paramList, varNewBool(false)); // delta + varLstAdd(paramList, varNewStrZ("12345678")); // cipherPass + + TEST_RESULT_BOOL( + backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - recopy, encrypt"); + TEST_RESULT_STR( + strPtr(strNewBuf(serverWrite)), "{\"out\":[2,9,32,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n", + " check result"); + bufUsedSet(serverWrite, 0); + } + + FUNCTION_HARNESS_RESULT_VOID(); +}