db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

The local command for backup is implemented entirely in C.

Browse Source 
The local process is now entirely migrated to C.  Since all major I/O operations are performed in the local process, the vast majority of I/O is now performed in C.

Contributed by David Steele, Cynthia Shang.
This commit is contained in:
David Steele 2019-07-25 14:34:16 -04:00
parent 54ec8f151e
commit 59f135340d
18 changed files with 990 additions and 917 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release.xml
View File

@ -33,6 +33,17 @@
</release-item>
</release-bug-list>
<release-improvement-list>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-contributor id="cynthia.shang"/>
</release-item-contributor-list>
<p>The <cmd>local</cmd> command for <cmd>backup</cmd> is implemented entirely in C.</p>
</release-item>
</release-improvement-list>
<release-development-list>
<release-item>
<release-item-contributor-list>

22
lib/pgBackRest/Backup/Backup.pm
View File

@ -336,13 +336,6 @@ sub processManifest
}
}
# Build the lsn start parameter to pass to the extra function
my $hStartLsnParam =
{
iWalId => defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,
iWalOffset => defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,
};
# Iterate all files in the manifest
foreach my $strRepoFile (
sort {sprintf("%016d-%s", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $b, MANIFEST_SUBKEY_SIZE), $b) cmp
@ -401,13 +394,13 @@ sub processManifest
# Queue for parallel backup
$oBackupProcess->queueJob(
$iHostConfigIdx, $strQueueKey, $strRepoFile, OP_BACKUP_FILE,
[$strDbFile, $strRepoFile, $lSize,
[$strDbFile, $bIgnoreMissing, $lSize,
$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, false),
cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $strBackupLabel, $bCompress,
cfgOption(CFGOPT_COMPRESS_LEVEL), $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile,
MANIFEST_SUBKEY_TIMESTAMP, false), $bIgnoreMissing,
cfgOption(CFGOPT_CHECKSUM_PAGE) && isChecksumPage($strRepoFile) ? $hStartLsnParam : undef,
cfgOption(CFGOPT_DELTA), defined($strReference) ? true : false],
cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false,
defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,
defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,
$strRepoFile, defined($strReference) ? true : false, $bCompress, cfgOption(CFGOPT_COMPRESS_LEVEL),
$strBackupLabel, cfgOption(CFGOPT_DELTA)],
{rParamSecure => $oBackupManifest->cipherPassSub() ? [$oBackupManifest->cipherPassSub()] : undef});
# Size and checksum will be removed and then verified later as a sanity check
@ -449,7 +442,8 @@ sub processManifest
{
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest, cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $hJob->{iHostConfigIdx}), false),
$hJob->{iProcessId}, @{$hJob->{rParam}}[0..4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,
$hJob->{iProcessId}, @{$hJob->{rParam}}[0], @{$hJob->{rParam}}[7], @{$hJob->{rParam}}[2], @{$hJob->{rParam}}[3],
@{$hJob->{rParam}}[4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,
$lManifestSaveCurrent);
}

244
lib/pgBackRest/Backup/File.pm
View File

@ -37,235 +37,6 @@ use constant BACKUP_FILE_SKIP => 3;
use constant BACKUP_FILE_NOOP => 4;
push @EXPORT, qw(BACKUP_FILE_NOOP);
####################################################################################################################################
# backupFile
####################################################################################################################################
sub backupFile
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbFile, # Database file to backup
$strRepoFile, # Location in the repository to copy to
$lSizeFile, # File size
$strChecksum, # File checksum to be checked
$bChecksumPage, # Should page checksums be calculated?
$strBackupLabel, # Label of current backup
$bCompress, # Compress destination file
$iCompressLevel, # Compress level
$lModificationTime, # File modification time
$bIgnoreMissing, # Is it OK if the file is missing?
$hExtraParam, # Parameter to pass to the extra function
$bDelta, # Is the delta option on?
$bHasReference, # Does the file exist in the repo in a prior backup in the set?
$strCipherPass, # Passphrase to access the repo file (undefined if repo not encrypted). This
# parameter must always be last in the parameter list to this function.
) =
logDebugParam
(
__PACKAGE__ . '::backupFile', \@_,
{name => 'strDbFile', trace => true},
{name => 'strRepoFile', trace => true},
{name => 'lSizeFile', trace => true},
{name => 'strChecksum', required => false, trace => true},
{name => 'bChecksumPage', trace => true},
{name => 'strBackupLabel', trace => true},
{name => 'bCompress', trace => true},
{name => 'iCompressLevel', trace => true},
{name => 'lModificationTime', trace => true},
{name => 'bIgnoreMissing', default => true, trace => true},
{name => 'hExtraParam', required => false, trace => true},
{name => 'bDelta', trace => true},
{name => 'bHasReference', trace => true},
{name => 'strCipherPass', required => false, trace => true},
);
my $oStorageRepo = storageRepo(); # Repo storage
my $iCopyResult = BACKUP_FILE_COPY; # Copy result
my $strCopyChecksum; # Copy checksum
my $rExtra; # Page checksum result
my $lCopySize; # Copy Size
my $lRepoSize; # Repo size
# Add compression suffix if needed
my $strFileOp = $strRepoFile . ($bCompress ? '.' . COMPRESS_EXT : '');
my $bCopy = true;
# If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo, else just
# check the repo.
if (defined($strChecksum))
{
# If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case then recopy.
if ($bDelta)
{
($strCopyChecksum, $lCopySize) = storageDb()->hashSize($strDbFile, {bIgnoreMissing => $bIgnoreMissing});
# If the DB file exists, then check the checksum
if (defined($strCopyChecksum))
{
$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);
# If the database file checksum and size are same and the file is in a prior backup, then no need to copy. If the
# checksum/size do not match, that is OK, just leave the copy result as COPY so the file will be copied to this
# backup.
if (!$bCopy && $bHasReference)
{
$iCopyResult = BACKUP_FILE_NOOP;
}
}
# Else the source file is missing from the database so skip this file
else
{
$iCopyResult = BACKUP_FILE_SKIP;
$bCopy = false;
}
}
# If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also test the
# checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, then there may be
# corruption in the repo, so recopy
if (!$bDelta || !$bHasReference)
{
# If this is a delta backup and the file is missing from the DB, then remove it from the repo (backupManifestUpdate will
# remove it from the manifest)
if ($iCopyResult == BACKUP_FILE_SKIP)
{
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}");
}
elsif (!$bDelta || !$bCopy)
{
# Add decompression
my $rhyFilter;
if ($bCompress)
{
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]});
}
# Get the checksum
($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize(
$oStorageRepo->openRead(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
{rhyFilter => $rhyFilter, strCipherPass => $strCipherPass}));
# Determine if the file needs to be recopied
$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);
# Set copy result
$iCopyResult = $bCopy ? BACKUP_FILE_RECOPY : BACKUP_FILE_CHECKSUM;
}
}
}
# Copy the file
if ($bCopy)
{
# Add size and sha filters
my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}];
# Add page checksum filter
if ($bChecksumPage)
{
# Determine which segment no this is by checking for a numeric extension. No extension means segment 0.
my $iSegmentNo = ($strDbFile =~ /\.[0-9]+$/) ? substr(($strDbFile =~ m/\.[0-9]+$/g)[0], 1) + 0 : 0;
push(
@{$rhyFilter},
{strClass => "pgBackRest::Backup::Filter::PageChecksum",
rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});
};
# Add compression
if ($bCompress)
{
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]});
}
# Else add protocol compression if the destination is not compressed and there is no encryption
elsif (!defined($strCipherPass))
{
push(
@{$rhyFilter},
{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]});
}
# Open the source file
my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing});
# Open the destination file
$rhyFilter = undef;
# Add protocol decompression if the destination is not compressed and there is no encryption
if (!$bCompress && !defined($strCipherPass))
{
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]});
}
my $oDestinationFileIo = $oStorageRepo->openWrite(
STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
{bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass});
$oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef);
# Copy the file
if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo))
{
# Get sha checksum and size
$strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);
$lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);
$lRepoSize = $oDestinationFileIo->result(COMMON_IO_HANDLE);
if (!defined($lRepoSize))
{
confess &log(ERROR, "REPO_SIZE IS NOT SET");
}
# Get results of page checksum validation
if ($bChecksumPage)
{
my $rExtraRaw = $oSourceFileIo->result("pgBackRest::Backup::Filter::PageChecksum");
$rExtra =
{
bValid => $rExtraRaw->{valid} ? true : false,
bAlign => $rExtraRaw->{align} ? true : false,
iyPageError => $rExtraRaw->{error},
};
}
}
# Else if source file is missing the database removed it
else
{
$iCopyResult = BACKUP_FILE_SKIP;
}
}
# If the file was copied get the repo size only if the storage can store the files with a different size than what was written.
# This has to be checked after the file is at rest because filesystem compression may affect the actual repo size and this
# cannot be calculated in stream.
#
# If the file was checksummed then get the size in all cases since we don't already have it.
if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) &&
$oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||
$iCopyResult == BACKUP_FILE_CHECKSUM)
{
$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->{size};
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iCopyResult', value => $iCopyResult},
{name => 'lCopySize', value => $lCopySize},
{name => 'lRepoSize', value => $lRepoSize},
{name => 'strCopyChecksum', value => $strCopyChecksum},
{name => 'rExtra', value => $rExtra},
);
}
push @EXPORT, qw(backupFile);
####################################################################################################################################
# backupManifestUpdate
####################################################################################################################################
@ -384,21 +155,22 @@ sub backupManifestUpdate
if ($bChecksumPage)
{
# The valid flag should be set
if (defined($rExtra->{bValid}))
if (defined($rExtra->{valid}))
{
# Store the valid flag
$oManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{bValid});
$oManifest->boolSet(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{valid});
# If the page was not valid
if (!$rExtra->{bValid})
if (!$rExtra->{valid})
{
# Check for a page misalignment
if ($lSizeCopy % PG_PAGE_SIZE != 0)
{
# Make sure the align flag was set, otherwise there is a bug
if (!defined($rExtra->{bAlign}) || $rExtra->{bAlign})
if (!defined($rExtra->{align}) || $rExtra->{align})
{
confess &log(ASSERT, 'bAlign flag should have been set for misaligned page');
confess &log(ASSERT, 'align flag should have been set for misaligned page');
}
# Emit a warning so the user knows something is amiss
@ -411,13 +183,13 @@ sub backupManifestUpdate
{
$oManifest->set(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,
dclone($rExtra->{iyPageError}));
dclone($rExtra->{error}));
# Build a pretty list of the page errors
my $strPageError;
my $iPageErrorTotal = 0;
foreach my $iyPage (@{$rExtra->{iyPageError}})
foreach my $iyPage (@{$rExtra->{error}})
{
$strPageError .= (defined($strPageError) ? ', ' : '');

24
lib/pgBackRest/Main.pm
View File

@ -107,30 +107,6 @@ sub main
cfgOption(CFGOPT_LOCK_PATH), cfgOption(CFGOPT_COMMAND), cfgOption(CFGOPT_STANZA, false), cfgOption(CFGOPT_PROCESS));
}
# Process local command
# --------------------------------------------------------------------------------------------------------------------------
elsif (cfgCommandTest(CFGCMD_LOCAL))
{
# Set log levels
cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);
logLevelSet(cfgOption(CFGOPT_LOG_LEVEL_FILE), OFF, cfgOption(CFGOPT_LOG_LEVEL_STDERR));
logFileSet(
storageLocal(),
cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgOption(CFGOPT_COMMAND)) . '-' .
lc(cfgCommandName(cfgCommandGet())) . '-' . sprintf("%03d", cfgOption(CFGOPT_PROCESS)));
# Load module dynamically
require pgBackRest::Protocol::Local::Minion;
pgBackRest::Protocol::Local::Minion->import();
# Create the local object
my $oLocal = new pgBackRest::Protocol::Local::Minion();
# Process local requests
$oLocal->process();
}
# Process check command
# --------------------------------------------------------------------------------------------------------------------------
elsif (cfgCommandTest(CFGCMD_CHECK))

69
lib/pgBackRest/Protocol/Local/Minion.pm
View File

@ -1,69 +0,0 @@
####################################################################################################################################
# PROTOCOL LOCAL MINION MODULE
####################################################################################################################################
package pgBackRest::Protocol::Local::Minion;
use parent 'pgBackRest::Protocol::Command::Minion';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Base::Master;
use pgBackRest::Protocol::Base::Minion;
use pgBackRest::Protocol::Command::Minion;
use pgBackRest::Protocol::Helper;
use pgBackRest::RestoreFile;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
# Init object and store variables
my $self = $class->SUPER::new(cfgCommandName(CFGCMD_LOCAL), cfgOption(CFGOPT_BUFFER_SIZE), cfgOption(CFGOPT_PROTOCOL_TIMEOUT));
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# init
####################################################################################################################################
sub init
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');
# Create anonymous subs for each command
my $hCommandMap =
{
&OP_BACKUP_FILE => sub {backupFile(@{shift()})},
# To be run after each command to keep the remote alive
&OP_POST => sub {protocolKeepAlive()},
};
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hCommandMap', value => $hCommandMap}
);
}
1;

1
libc/Makefile.PL
View File

@ -42,7 +42,6 @@ my @stryCFile =
(
'LibC.c',
'command/backup/pageChecksum.c',
'command/command.c',
'common/compress/gzip/common.c',
'common/compress/gzip/compress.c',

17
libc/xs/storage/storage.xsh
View File

@ -1,7 +1,6 @@
/***********************************************************************************************************************************
Storage XS Header
***********************************************************************************************************************************/
#include "command/backup/pageChecksum.h"
#include "common/assert.h"
#include "common/compress/gzip/compress.h"
#include "common/compress/gzip/decompress.h"
@ -127,14 +126,6 @@ storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const Strin
{
ioFilterGroupAdd(filterGroup, ioSizeNew());
}
else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum"))
{
ioFilterGroupAdd(
filterGroup,
pageChecksumNew(
varUInt64Force(varLstGet(paramList, 0)), PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT,
(uint64_t)varUIntForce(varLstGet(paramList, 1)) << 32 | varUIntForce(varLstGet(paramList, 2))));
}
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gzip"))
{
if (strEqZ(varStr(varLstGet(paramList, 0)), "compress"))
@ -167,10 +158,6 @@ storageFilterXsResult(const IoFilterGroup *filterGroup, const String *filter)
{
result = ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR);
}
else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum"))
{
result = ioFilterGroupResult(filterGroup, PAGE_CHECKSUM_FILTER_TYPE_STR);
}
else
THROW_FMT(AssertError, "unable to get result for invalid filter '%s'", strPtr(filter));
@ -202,10 +189,6 @@ storageFilterXsResultAll(const IoFilterGroup *filterGroup)
{
filterPerl = strNew("pgBackRest::Common::Io::Handle");
}
else if (strEq(filter, PAGE_CHECKSUM_FILTER_TYPE_STR))
{
filterPerl = strNew("pgBackRest::Backup::Filter::PageChecksum");
}
if (filterPerl != NULL)
{

12
src/Makefile.in
View File

@ -52,7 +52,9 @@ SRCS = \
command/archive/push/protocol.c \
command/archive/push/push.c \
command/backup/common.c \
command/backup/file.c \
command/backup/pageChecksum.c \
command/backup/protocol.c \
command/expire/expire.c \
command/help/help.c \
command/info/info.c \
@ -219,9 +221,15 @@ command/archive/push/push.o: command/archive/push/push.c build.auto.h command/ar
command/backup/common.o: command/backup/common.c build.auto.h command/backup/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/common.c -o command/backup/common.o
command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/common.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/file.c -o command/backup/file.o
command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h command/backup/pageChecksum.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/pageChecksum.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/pageChecksum.c -o command/backup/pageChecksum.o
command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/protocol.c -o command/backup/protocol.o
command/command.o: command/command.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/tls/client.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/command.c -o command/command.o
@ -237,7 +245,7 @@ command/help/help.o: command/help/help.c build.auto.h common/assert.h common/deb
command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/info/info.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o
command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/backup/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/local/local.c -o command/local/local.o
command/remote/remote.o: command/remote/remote.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/remote/protocol.h
@ -438,7 +446,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o
postgres/interface.o: postgres/interface.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/version.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h

252
src/command/backup/file.c Normal file
View File

@ -0,0 +1,252 @@
/***********************************************************************************************************************************
Backup File
***********************************************************************************************************************************/
#include "build.auto.h"
#include <string.h>
#include "command/backup/file.h"
#include "command/backup/pageChecksum.h"
#include "common/compress/gzip/common.h"
#include "common/compress/gzip/compress.h"
#include "common/compress/gzip/decompress.h"
#include "common/crypto/cipherBlock.h"
#include "common/crypto/hash.h"
#include "common/debug.h"
#include "common/io/filter/group.h"
#include "common/io/filter/size.h"
#include "common/io/io.h"
#include "common/log.h"
#include "common/regExp.h"
#include "common/type/convert.h"
#include "postgres/interface.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Helper functions
***********************************************************************************************************************************/
static unsigned int
segmentNumber(const String *pgFile)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, pgFile);
FUNCTION_TEST_END();
// Determine which segment number this is by checking for a numeric extension. No extension means segment 0.
FUNCTION_TEST_RETURN(regExpMatchOne(STRDEF("\\.[0-9]+$"), pgFile) ? cvtZToUInt(strrchr(strPtr(pgFile), '.') + 1) : 0);
}
/***********************************************************************************************************************************
Copy a file from the PostgreSQL data directory to the repository
***********************************************************************************************************************************/
BackupFileResult
backupFile(
const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage,
uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress,
unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, pgFile); // Database file to copy to the repo
FUNCTION_LOG_PARAM(BOOL, pgFileIgnoreMissing); // Is it OK if the database file is missing?
FUNCTION_LOG_PARAM(UINT64, pgFileSize); // Size of the database file
FUNCTION_LOG_PARAM(STRING, pgFileChecksum); // Checksum to verify the database file
FUNCTION_LOG_PARAM(BOOL, pgFileChecksumPage); // Should page checksums be validated
FUNCTION_LOG_PARAM(UINT64, pgFileChecksumPageLsnLimit); // Upper LSN limit to which page checksums must be valid
FUNCTION_LOG_PARAM(STRING, repoFile); // Destination in the repo to copy the pg file
FUNCTION_LOG_PARAM(BOOL, repoFileHasReference); // Does the repo file exists in a prior backup in the set?
FUNCTION_LOG_PARAM(BOOL, repoFileCompress); // Compress destination file
FUNCTION_LOG_PARAM(UINT, repoFileCompressLevel); // Compression level for destination file
FUNCTION_LOG_PARAM(STRING, backupLabel); // Label of current backup
FUNCTION_LOG_PARAM(BOOL, delta); // Is the delta option on?
FUNCTION_LOG_PARAM(ENUM, cipherType); // Encryption type
FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted
FUNCTION_LOG_END();
ASSERT(pgFile != NULL);
ASSERT(repoFile != NULL);
ASSERT(backupLabel != NULL);
ASSERT((cipherType == cipherTypeNone && cipherPass == NULL) || (cipherType != cipherTypeNone && cipherPass != NULL));
// Backup file results
BackupFileResult result = {.backupCopyResult = backupCopyResultCopy};
MEM_CONTEXT_TEMP_BEGIN()
{
// Generate complete repo path and add compression extension if needed
const String *repoPathFile = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(backupLabel), strPtr(repoFile), repoFileCompress ? "." GZIP_EXT : "");
// If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo, else
// just check the repo.
if (pgFileChecksum != NULL)
{
// Does the file in pg match the checksum and size passed?
bool pgFileMatch = false;
// If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case then
// recopy.
if (delta)
{
// Generate checksum/size for the pg file
IoRead *read = storageReadIo(storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
{
const String *pgTestChecksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR));
// Does the pg file match?
if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum))
{
pgFileMatch = true;
// If it matches and is a reference to a previous backup then no need to copy the file
if (repoFileHasReference)
{
memContextSwitch(MEM_CONTEXT_OLD());
result.backupCopyResult = backupCopyResultNoOp;
result.copySize = pgTestSize;
result.copyChecksum = strDup(pgTestChecksum);
memContextSwitch(MEM_CONTEXT_TEMP());
}
}
}
// Else the source file is missing from the database so skip this file
else
result.backupCopyResult = backupCopyResultSkip;
}
// If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also test the
// checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, then there may
// be corruption in the repo, so recopy
if (!delta || !repoFileHasReference)
{
// If this is a delta backup and the file is missing from the DB, then remove it from the repo (backupManifestUpdate
// will remove it from the manifest)
if (result.backupCopyResult == backupCopyResultSkip)
{
storageRemoveNP(storageRepoWrite(), repoPathFile);
}
else if (!delta || pgFileMatch)
{
// Generate checksum/size for the repo file
IoRead *read = storageReadIo(storageNewReadNP(storageRepo(), repoPathFile));
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(read), cipherBlockNew(cipherModeDecrypt, cipherType, BUFSTR(cipherPass), NULL));
}
if (repoFileCompress)
ioFilterGroupAdd(ioReadFilterGroup(read), gzipDecompressNew(false));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
ioReadDrain(read);
// Test checksum/size
const String *pgTestChecksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR));
// No need to recopy if checksum/size match
if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum))
{
memContextSwitch(MEM_CONTEXT_OLD());
result.backupCopyResult = backupCopyResultChecksum;
result.copySize = pgTestSize;
result.copyChecksum = strDup(pgTestChecksum);
memContextSwitch(MEM_CONTEXT_TEMP());
}
// Else recopy when repo file is not as expected
else
result.backupCopyResult = backupCopyResultReCopy;
}
}
}
// Copy the file
if (result.backupCopyResult == backupCopyResultCopy || result.backupCopyResult == backupCopyResultReCopy)
{
// Is the file compressible during the copy?
bool compressible = !repoFileCompress && cipherType == cipherTypeNone;
// Setup pg file for read
StorageRead *read = storageNewReadP(
storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
// Add page checksum filter
if (pgFileChecksumPage)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), pageChecksumNew(segmentNumber(pgFile), PG_SEGMENT_PAGE_DEFAULT,
PG_PAGE_SIZE_DEFAULT, pgFileChecksumPageLsnLimit));
}
// Add compression
if (repoFileCompress)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzipCompressNew((int)repoFileCompressLevel, false));
// If there is a cipher then add the encrypt filter
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(
storageReadIo(read)), cipherBlockNew(cipherModeEncrypt, cipherType, BUFSTR(cipherPass), NULL));
}
// Setup the repo file for write
StorageWrite *write = storageNewWriteP(storageRepoWrite(), repoPathFile, .compressible = compressible);
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), ioSizeNew());
// Open the source and destination and copy the file
if (storageCopy(read, write))
{
memContextSwitch(MEM_CONTEXT_OLD());
// Get sizes and checksum
result.copySize = varUInt64Force(
ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE_STR));
result.copyChecksum = strDup(
varStr(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR)));
result.repoSize =
varUInt64Force(ioFilterGroupResult(ioWriteFilterGroup(storageWriteIo(write)), SIZE_FILTER_TYPE_STR));
// Get results of page checksum validation
if (pgFileChecksumPage)
{
result.pageChecksumResult = kvDup(
varKv(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE_STR)));
}
memContextSwitch(MEM_CONTEXT_TEMP());
}
// Else if source file is missing and the read setup indicated ignore a missing file, the database removed it so skip it
else
result.backupCopyResult = backupCopyResultSkip;
}
// If the file was copied get the repo size only if the storage can store the files with a different size than what was
// written. This has to be checked after the file is at rest because filesystem compression may affect the actual repo size
// and this cannot be calculated in stream.
//
// If the file was checksummed then get the size in all cases since we don't already have it.
if (((result.backupCopyResult == backupCopyResultCopy || result.backupCopyResult == backupCopyResultReCopy) &&
storageFeature(storageRepo(), storageFeatureCompress)) ||
result.backupCopyResult == backupCopyResultChecksum)
{
result.repoSize = storageInfoNP(storageRepo(), repoPathFile).size;
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BACKUP_FILE_RESULT, result);
}

48
src/command/backup/file.h Normal file
View File

@ -0,0 +1,48 @@
/***********************************************************************************************************************************
Backup File
***********************************************************************************************************************************/
#ifndef COMMAND_BACKUP_FILE_H
#define COMMAND_BACKUP_FILE_H
#include "common/crypto/common.h"
#include "common/type/keyValue.h"
/***********************************************************************************************************************************
Backup file types
***********************************************************************************************************************************/
typedef enum
{
backupCopyResultChecksum,
backupCopyResultCopy,
backupCopyResultReCopy,
backupCopyResultSkip,
backupCopyResultNoOp,
} BackupCopyResult;
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
typedef struct BackupFileResult
{
BackupCopyResult backupCopyResult;
uint64_t copySize;
String *copyChecksum;
uint64_t repoSize;
KeyValue *pageChecksumResult;
} BackupFileResult;
BackupFileResult backupFile(
const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage,
uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress,
unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_BACKUP_FILE_RESULT_TYPE \
BackupFileResult
#define FUNCTION_LOG_BACKUP_FILE_RESULT_FORMAT(value, buffer, bufferSize) \
objToLog(&value, "BackupFileResult", buffer, bufferSize)
#endif

67
src/command/backup/protocol.c Normal file
View File

@ -0,0 +1,67 @@
/***********************************************************************************************************************************
Backup Protocol Handler
***********************************************************************************************************************************/
#include "build.auto.h"
#include "command/backup/file.h"
#include "command/backup/protocol.h"
#include "common/debug.h"
#include "common/io/io.h"
#include "common/log.h"
#include "common/memContext.h"
#include "config/config.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_COMMAND_BACKUP_FILE_STR, PROTOCOL_COMMAND_BACKUP_FILE);
/***********************************************************************************************************************************
Process protocol requests
***********************************************************************************************************************************/
bool
backupProtocol(const String *command, const VariantList *paramList, ProtocolServer *server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, command);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(command != NULL);
// Attempt to satisfy the request -- we may get requests that are meant for other handlers
bool found = true;
MEM_CONTEXT_TEMP_BEGIN()
{
if (strEq(command, PROTOCOL_COMMAND_BACKUP_FILE_STR))
{
// Backup the file
BackupFileResult result = backupFile(
varStr(varLstGet(paramList, 0)), varBoolForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
varStr(varLstGet(paramList, 3)), varBoolForce(varLstGet(paramList, 4)),
varUInt64(varLstGet(paramList, 5)) << 32 | varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)),
varBoolForce(varLstGet(paramList, 8)), varBoolForce(varLstGet(paramList, 9)),
varUIntForce(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)), varBoolForce(varLstGet(paramList, 12)),
varLstSize(paramList) == 14 ? cipherTypeAes256Cbc : cipherTypeNone,
varLstSize(paramList) == 14 ? varStr(varLstGet(paramList, 13)) : NULL);
// Return backup result
VariantList *resultList = varLstNew();
varLstAdd(resultList, varNewUInt(result.backupCopyResult));
varLstAdd(resultList, varNewUInt64(result.copySize));
varLstAdd(resultList, varNewUInt64(result.repoSize));
varLstAdd(resultList, varNewStr(result.copyChecksum));
varLstAdd(resultList, result.pageChecksumResult != NULL ? varNewKv(result.pageChecksumResult) : NULL);
protocolServerResponse(server, varNewVarLst(resultList));
}
else
found = false;
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, found);
}

22
src/command/backup/protocol.h Normal file
View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
Backup Protocol Handler
***********************************************************************************************************************************/
#ifndef COMMAND_BACKUP_PROTOCOL_H
#define COMMAND_BACKUP_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_COMMAND_BACKUP_FILE "backupFile"
STRING_DECLARE(PROTOCOL_COMMAND_BACKUP_FILE_STR);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
bool backupProtocol(const String *command, const VariantList *paramList, ProtocolServer *server);
#endif

2
src/command/local/local.c
View File

@ -5,6 +5,7 @@ Local Command
#include "command/archive/get/protocol.h"
#include "command/archive/push/protocol.h"
#include "command/backup/protocol.h"
#include "command/restore/protocol.h"
#include "common/debug.h"
#include "common/io/handleRead.h"
@ -34,6 +35,7 @@ cmdLocal(int handleRead, int handleWrite)
ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_LOCAL_STR, read, write);
protocolServerHandlerAdd(server, archiveGetProtocol);
protocolServerHandlerAdd(server, archivePushProtocol);
protocolServerHandlerAdd(server, backupProtocol);
protocolServerHandlerAdd(server, restoreProtocol);
protocolServerProcess(server);
}

10
src/main.c
View File

@ -158,15 +158,7 @@ main(int argListSize, const char *argList[])
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdLocal:
{
if (strEq(cfgOptionStr(cfgOptCommand), CFGCMD_ARCHIVE_GET_ASYNC_STR) ||
strEq(cfgOptionStr(cfgOptCommand), CFGCMD_ARCHIVE_PUSH_ASYNC_STR) ||
strEq(cfgOptionStr(cfgOptCommand), CFGCMD_RESTORE_STR))
{
cmdLocal(STDIN_FILENO, STDOUT_FILENO);
}
else
perlExec();
cmdLocal(STDIN_FILENO, STDOUT_FILENO);
break;
}

311
src/perl/embed.auto.c
View File

@ -1165,12 +1165,6 @@ static const EmbeddedModule embeddedModule[] =
"}\n"
"}\n"
"\n\n"
"my $hStartLsnParam =\n"
"{\n"
"iWalId => defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,\n"
"iWalOffset => defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,\n"
"};\n"
"\n\n"
"foreach my $strRepoFile (\n"
"sort {sprintf(\"%016d-%s\", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $b, MANIFEST_SUBKEY_SIZE), $b) cmp\n"
"sprintf(\"%016d-%s\", $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $a, MANIFEST_SUBKEY_SIZE), $a)}\n"
@ -1219,13 +1213,13 @@ static const EmbeddedModule embeddedModule[] =
"\n\n"
"$oBackupProcess->queueJob(\n"
"$iHostConfigIdx, $strQueueKey, $strRepoFile, OP_BACKUP_FILE,\n"
"[$strDbFile, $strRepoFile, $lSize,\n"
"[$strDbFile, $bIgnoreMissing, $lSize,\n"
"$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, false),\n"
"cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false, $strBackupLabel, $bCompress,\n"
"cfgOption(CFGOPT_COMPRESS_LEVEL), $oBackupManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile,\n"
"MANIFEST_SUBKEY_TIMESTAMP, false), $bIgnoreMissing,\n"
"cfgOption(CFGOPT_CHECKSUM_PAGE) && isChecksumPage($strRepoFile) ? $hStartLsnParam : undef,\n"
"cfgOption(CFGOPT_DELTA), defined($strReference) ? true : false],\n"
"cfgOption(CFGOPT_CHECKSUM_PAGE) ? isChecksumPage($strRepoFile) : false,\n"
"defined($strLsnStart) ? hex((split('/', $strLsnStart))[0]) : 0xFFFFFFFF,\n"
"defined($strLsnStart) ? hex((split('/', $strLsnStart))[1]) : 0xFFFFFFFF,\n"
"$strRepoFile, defined($strReference) ? true : false, $bCompress, cfgOption(CFGOPT_COMPRESS_LEVEL),\n"
"$strBackupLabel, cfgOption(CFGOPT_DELTA)],\n"
"{rParamSecure => $oBackupManifest->cipherPassSub() ? [$oBackupManifest->cipherPassSub()] : undef});\n"
"\n\n"
"$oBackupManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_SIZE);\n"
@ -1260,7 +1254,8 @@ static const EmbeddedModule embeddedModule[] =
"{\n"
"($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(\n"
"$oBackupManifest, cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $hJob->{iHostConfigIdx}), false),\n"
"$hJob->{iProcessId}, @{$hJob->{rParam}}[0..4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,\n"
"$hJob->{iProcessId}, @{$hJob->{rParam}}[0], @{$hJob->{rParam}}[7], @{$hJob->{rParam}}[2], @{$hJob->{rParam}}[3],\n"
"@{$hJob->{rParam}}[4], @{$hJob->{rResult}}, $lSizeTotal, $lSizeCurrent, $lManifestSaveSize,\n"
"$lManifestSaveCurrent);\n"
"}\n"
"\n\n\n"
@ -2092,204 +2087,6 @@ static const EmbeddedModule embeddedModule[] =
"use constant BACKUP_FILE_NOOP => 4;\n"
"push @EXPORT, qw(BACKUP_FILE_NOOP);\n"
"\n\n\n\n"
"sub backupFile\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strDbFile,\n"
"$strRepoFile,\n"
"$lSizeFile,\n"
"$strChecksum,\n"
"$bChecksumPage,\n"
"$strBackupLabel,\n"
"$bCompress,\n"
"$iCompressLevel,\n"
"$lModificationTime,\n"
"$bIgnoreMissing,\n"
"$hExtraParam,\n"
"$bDelta,\n"
"$bHasReference,\n"
"$strCipherPass,\n"
"\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::backupFile', \\@_,\n"
"{name => 'strDbFile', trace => true},\n"
"{name => 'strRepoFile', trace => true},\n"
"{name => 'lSizeFile', trace => true},\n"
"{name => 'strChecksum', required => false, trace => true},\n"
"{name => 'bChecksumPage', trace => true},\n"
"{name => 'strBackupLabel', trace => true},\n"
"{name => 'bCompress', trace => true},\n"
"{name => 'iCompressLevel', trace => true},\n"
"{name => 'lModificationTime', trace => true},\n"
"{name => 'bIgnoreMissing', default => true, trace => true},\n"
"{name => 'hExtraParam', required => false, trace => true},\n"
"{name => 'bDelta', trace => true},\n"
"{name => 'bHasReference', trace => true},\n"
"{name => 'strCipherPass', required => false, trace => true},\n"
");\n"
"\n"
"my $oStorageRepo = storageRepo();\n"
"my $iCopyResult = BACKUP_FILE_COPY;\n"
"my $strCopyChecksum;\n"
"my $rExtra;\n"
"my $lCopySize;\n"
"my $lRepoSize;\n"
"\n\n"
"my $strFileOp = $strRepoFile . ($bCompress ? '.' . COMPRESS_EXT : '');\n"
"\n"
"my $bCopy = true;\n"
"\n\n\n"
"if (defined($strChecksum))\n"
"{\n"
"\n"
"if ($bDelta)\n"
"{\n"
"($strCopyChecksum, $lCopySize) = storageDb()->hashSize($strDbFile, {bIgnoreMissing => $bIgnoreMissing});\n"
"\n\n"
"if (defined($strCopyChecksum))\n"
"{\n"
"$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);\n"
"\n\n\n\n"
"if (!$bCopy && $bHasReference)\n"
"{\n"
"$iCopyResult = BACKUP_FILE_NOOP;\n"
"}\n"
"}\n"
"\n"
"else\n"
"{\n"
"$iCopyResult = BACKUP_FILE_SKIP;\n"
"$bCopy = false;\n"
"}\n"
"}\n"
"\n\n\n\n"
"if (!$bDelta || !$bHasReference)\n"
"{\n"
"\n\n"
"if ($iCopyResult == BACKUP_FILE_SKIP)\n"
"{\n"
"$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\");\n"
"}\n"
"elsif (!$bDelta || !$bCopy)\n"
"{\n"
"\n"
"my $rhyFilter;\n"
"\n"
"if ($bCompress)\n"
"{\n"
"push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]});\n"
"}\n"
"\n\n"
"($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize(\n"
"$oStorageRepo->openRead(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\",\n"
"{rhyFilter => $rhyFilter, strCipherPass => $strCipherPass}));\n"
"\n\n"
"$bCopy = !($strCopyChecksum eq $strChecksum && $lCopySize == $lSizeFile);\n"
"\n\n"
"$iCopyResult = $bCopy ? BACKUP_FILE_RECOPY : BACKUP_FILE_CHECKSUM;\n"
"}\n"
"}\n"
"}\n"
"\n\n"
"if ($bCopy)\n"
"{\n"
"\n"
"my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}];\n"
"\n\n"
"if ($bChecksumPage)\n"
"{\n"
"\n"
"my $iSegmentNo = ($strDbFile =~ /\\.[0-9]+$/) ? substr(($strDbFile =~ m/\\.[0-9]+$/g)[0], 1) + 0 : 0;\n"
"\n"
"push(\n"
"@{$rhyFilter},\n"
"{strClass => \"pgBackRest::Backup::Filter::PageChecksum\",\n"
"rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});\n"
"};\n"
"\n\n"
"if ($bCompress)\n"
"{\n"
"push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]});\n"
"}\n"
"\n"
"elsif (!defined($strCipherPass))\n"
"{\n"
"push(\n"
"@{$rhyFilter},\n"
"{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]});\n"
"}\n"
"\n\n"
"my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing});\n"
"\n\n"
"$rhyFilter = undef;\n"
"\n\n"
"if (!$bCompress && !defined($strCipherPass))\n"
"{\n"
"push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]});\n"
"}\n"
"\n"
"my $oDestinationFileIo = $oStorageRepo->openWrite(\n"
"STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\",\n"
"{bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass});\n"
"\n"
"$oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef);\n"
"\n\n"
"if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo))\n"
"{\n"
"\n"
"$strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);\n"
"$lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);\n"
"$lRepoSize = $oDestinationFileIo->result(COMMON_IO_HANDLE);\n"
"\n"
"if (!defined($lRepoSize))\n"
"{\n"
"confess &log(ERROR, \"REPO_SIZE IS NOT SET\");\n"
"}\n"
"\n\n"
"if ($bChecksumPage)\n"
"{\n"
"my $rExtraRaw = $oSourceFileIo->result(\"pgBackRest::Backup::Filter::PageChecksum\");\n"
"\n"
"$rExtra =\n"
"{\n"
"bValid => $rExtraRaw->{valid} ? true : false,\n"
"bAlign => $rExtraRaw->{align} ? true : false,\n"
"iyPageError => $rExtraRaw->{error},\n"
"};\n"
"}\n"
"}\n"
"\n"
"else\n"
"{\n"
"$iCopyResult = BACKUP_FILE_SKIP;\n"
"}\n"
"}\n"
"\n\n\n\n\n\n"
"if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) &&\n"
"$oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||\n"
"$iCopyResult == BACKUP_FILE_CHECKSUM)\n"
"{\n"
"$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\"))->{size};\n"
"}\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'iCopyResult', value => $iCopyResult},\n"
"{name => 'lCopySize', value => $lCopySize},\n"
"{name => 'lRepoSize', value => $lRepoSize},\n"
"{name => 'strCopyChecksum', value => $strCopyChecksum},\n"
"{name => 'rExtra', value => $rExtra},\n"
");\n"
"}\n"
"\n"
"push @EXPORT, qw(backupFile);\n"
"\n\n\n\n"
"sub backupManifestUpdate\n"
"{\n"
"\n"
@ -2397,20 +2194,21 @@ static const EmbeddedModule embeddedModule[] =
"if ($bChecksumPage)\n"
"{\n"
"\n"
"if (defined($rExtra->{bValid}))\n"
"if (defined($rExtra->{valid}))\n"
"{\n"
"\n"
"$oManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{bValid});\n"
"$oManifest->boolSet(\n"
"MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{valid});\n"
"\n\n"
"if (!$rExtra->{bValid})\n"
"if (!$rExtra->{valid})\n"
"{\n"
"\n"
"if ($lSizeCopy % PG_PAGE_SIZE != 0)\n"
"{\n"
"\n"
"if (!defined($rExtra->{bAlign}) || $rExtra->{bAlign})\n"
"if (!defined($rExtra->{align}) || $rExtra->{align})\n"
"{\n"
"confess &log(ASSERT, 'bAlign flag should have been set for misaligned page');\n"
"confess &log(ASSERT, 'align flag should have been set for misaligned page');\n"
"}\n"
"\n\n"
"&log(WARN,\n"
@ -2422,12 +2220,12 @@ static const EmbeddedModule embeddedModule[] =
"{\n"
"$oManifest->set(\n"
"MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,\n"
"dclone($rExtra->{iyPageError}));\n"
"dclone($rExtra->{error}));\n"
"\n\n"
"my $strPageError;\n"
"my $iPageErrorTotal = 0;\n"
"\n"
"foreach my $iyPage (@{$rExtra->{iyPageError}})\n"
"foreach my $iyPage (@{$rExtra->{error}})\n"
"{\n"
"$strPageError .= (defined($strPageError) ? ', ' : '');\n"
"\n\n"
@ -8681,25 +8479,6 @@ static const EmbeddedModule embeddedModule[] =
"cfgOption(CFGOPT_LOCK_PATH), cfgOption(CFGOPT_COMMAND), cfgOption(CFGOPT_STANZA, false), cfgOption(CFGOPT_PROCESS));\n"
"}\n"
"\n\n\n"
"elsif (cfgCommandTest(CFGCMD_LOCAL))\n"
"{\n"
"\n"
"cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);\n"
"logLevelSet(cfgOption(CFGOPT_LOG_LEVEL_FILE), OFF, cfgOption(CFGOPT_LOG_LEVEL_STDERR));\n"
"\n"
"logFileSet(\n"
"storageLocal(),\n"
"cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgOption(CFGOPT_COMMAND)) . '-' .\n"
"lc(cfgCommandName(cfgCommandGet())) . '-' . sprintf(\"%03d\", cfgOption(CFGOPT_PROCESS)));\n"
"\n\n"
"require pgBackRest::Protocol::Local::Minion;\n"
"pgBackRest::Protocol::Local::Minion->import();\n"
"\n\n"
"my $oLocal = new pgBackRest::Protocol::Local::Minion();\n"
"\n\n"
"$oLocal->process();\n"
"}\n"
"\n\n\n"
"elsif (cfgCommandTest(CFGCMD_CHECK))\n"
"{\n"
"\n"
@ -11260,64 +11039,6 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"1;\n"
},
{
.name = "pgBackRest/Protocol/Local/Minion.pm",
.data =
"\n\n\n"
"package pgBackRest::Protocol::Local::Minion;\n"
"use parent 'pgBackRest::Protocol::Command::Minion';\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"\n"
"use pgBackRest::Backup::File;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Protocol::Base::Master;\n"
"use pgBackRest::Protocol::Base::Minion;\n"
"use pgBackRest::Protocol::Command::Minion;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::RestoreFile;\n"
"\n\n\n\n"
"sub new\n"
"{\n"
"my $class = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');\n"
"\n\n"
"my $self = $class->SUPER::new(cfgCommandName(CFGCMD_LOCAL), cfgOption(CFGOPT_BUFFER_SIZE), cfgOption(CFGOPT_PROTOCOL_TIMEOUT));\n"
"bless $self, $class;\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'self', value => $self}\n"
");\n"
"}\n"
"\n\n\n\n"
"sub init\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');\n"
"\n\n"
"my $hCommandMap =\n"
"{\n"
"&OP_BACKUP_FILE => sub {backupFile(@{shift()})},\n"
"\n\n"
"&OP_POST => sub {protocolKeepAlive()},\n"
"};\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'hCommandMap', value => $hCommandMap}\n"
");\n"
"}\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/Protocol/Local/Process.pm",
.data =

9
test/define.yaml
View File

@ -573,6 +573,15 @@ unit:
command/backup/common: full
command/backup/pageChecksum: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: backup
total: 3
coverage:
command/backup/file: full
command/backup/protocol: full
storage/storage: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: command
total: 1

329
test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm
View File

@ -125,16 +125,8 @@ sub run
if ($self->begin('backupFile(), backupManifestUpdate()'))
{
#---------------------------------------------------------------------------------------------------------------------------
# Copy pg_control and confirm manifestUpdate does not save the manifest yet
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($self->{strPgControl}, MANIFEST_FILE_PGCONTROL, $lPgControlSize, undef, false, $strBackupLabel, false,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lPgControlTime, true, undef, false, false, undef);
$self->testResult(sub {storageTest()->exists($strPgControlRepo)}, true, 'pg_control file exists in repo');
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strPgControlHash &&
$lResultCopySize == $lPgControlSize && $lResultRepoSize == $lPgControlSize), true,
'pg_control file copied to repo successfully');
# Create backup path so manifest can be saved
storageRepo->pathCreate(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/$strBackupLabel"));
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
@ -145,15 +137,15 @@ sub run
$lPgControlSize,
undef,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
BACKUP_FILE_COPY,
8192,
8192,
$strPgControlHash,
undef,
16785408,
0,
167854,
0);
# Accumulators should be same size as pg_control
$self->testResult(($lSizeCurrent == $lPgControlSize && $lManifestSaveCurrent == $lPgControlSize), true,
@ -169,18 +161,6 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# No prior checksum, no compression, no page checksum, no extra, no delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, undef, false, $strBackupLabel, false,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef);
$self->testResult(sub {storageTest()->exists($strFileRepo)}, true, 'non-compressed file exists in repo');
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true,
'file copied to repo successfully');
$self->testResult(sub {storageRepo()->exists("${strFileRepo}.gz")}, false, "${strFileRepo}.gz missing");
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
@ -190,15 +170,15 @@ sub run
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
BACKUP_FILE_COPY,
16777216,
16777216,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
8192,
167854,
8192);
# Accumulator includes size of pg_control and file. Manifest saved so ManifestSaveCurrent returns to 0
$self->testResult(($lSizeCurrent == ($lPgControlSize + $lFileSize) && $lManifestSaveCurrent == 0), true,
@ -213,29 +193,9 @@ sub run
$self->testResult(
sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, 'backup.manifest.copy missing in repo');
storageTest()->remove($strFileRepo);
storageTest()->remove($strPgControlRepo);
#---------------------------------------------------------------------------------------------------------------------------
# Build the lsn start parameter to pass to the extra function
my $hStartLsnParam =
{
iWalId => 0xFFFF,
iWalOffset => 0xFFFF,
};
# No prior checksum, yes compression, yes page checksum, yes extra, no delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true, $strBackupLabel, true,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, $hStartLsnParam, false, false, undef);
$self->testResult(sub {storageTest()->exists("$strFileRepo.gz")}, true, 'compressed file exists in repo');
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultRepoSize == $lRepoFileCompressSize && $rResultExtra->{bValid}), true, 'file copied to repo successfully');
# Only the compressed version of the file exists
$self->testResult(sub {storageRepo()->exists("$strFileRepo")}, false, "only compressed version exists");
# Set up page checksum result
$rResultExtra = {'valid' => true,'align' => true};
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
@ -246,15 +206,15 @@ sub run
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
BACKUP_FILE_COPY,
16777216,
3646899,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
16785408,
16785408,
167854,
0);
# File is compressed in repo so make sure repo-size added to manifest
$self->testResult(sub {$oBackupManifest->test(
@ -264,45 +224,15 @@ sub run
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rResultExtra->{bValid})},
true, "checksum page set");
# Save the compressed file for later test
executeTest('mv ' . "$strFileRepo.gz $strFileRepo.gz.SAVE");
#---------------------------------------------------------------------------------------------------------------------------
# Add a segment number for bChecksumPage code coverage
executeTest('cp ' . "$strFileDb $strFileDb.1");
# No prior checksum, no compression, yes page checksum, yes extra, no delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, undef, true, $strBackupLabel, false,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, $hStartLsnParam, false, false, undef);
$self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, true, 'non-compressed segment file exists in repo');
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultRepoSize == $lFileSize && $rResultExtra->{bValid}), true, 'segment file copied to repo successfully');
# Set a section in the manifest to ensure it is removed in the next test
$oBackupManifest->set(
MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1", MANIFEST_SUBKEY_CHECKSUM, $strResultCopyChecksum);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.1")},
true, MANIFEST_TARGET_PGDATA . "/$strFileName.1 section exists in manifest");
true, MANIFEST_TARGET_PGDATA . "/$strFileName.1 section exists in manifest - skip file");
#---------------------------------------------------------------------------------------------------------------------------
# Remove the db file and try to back it up
storageTest()->remove("$strFileDb.1");
# No prior checksum, no compression, no page checksum, no extra, No delta, no hasReference, no db file
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, undef, false, $strBackupLabel,
false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_SKIP && !defined($strResultCopyChecksum) &&
!defined($lResultRepoSize) && !defined($lResultCopySize)), true, "db file missing - $strRepoFile.1 file skipped");
# Delta not set so file still exists in repo
$self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, true, ' delta not set - file exists in repo');
# Removed db file is removed from manifest
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
@ -312,30 +242,23 @@ sub run
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
BACKUP_FILE_SKIP,
undef,
undef,
undef,
undef,
16785408,
33562624,
167854,
0);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")},
false, " $strRepoFile.1 section removed from manifest");
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference, no db file
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile("$strFileDb.1", MANIFEST_TARGET_PGDATA . "/$strFileName.1", $lFileSize, $strFileHash, false, $strBackupLabel,
false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef);
# Add back the section
$oBackupManifest->set(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1");
$self->testResult(($iResultCopyResult == BACKUP_FILE_SKIP && !defined($strResultCopyChecksum) &&
!defined($lResultRepoSize)), true, "db file missing - delta $strRepoFile.1 file skipped");
$self->testResult(sub {storageTest()->exists("$strFileRepo.1")}, false, ' delta set - file removed from repo');
# Code path for host not defined for logged message of skipped file
# Code coverage for code path when host not defined for logged message of skipped file
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
undef,
@ -345,100 +268,21 @@ sub run
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
BACKUP_FILE_SKIP,
undef,
undef,
undef,
undef,
16785408,
50339840,
167854,
0);
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference, no db file,
# do not ignoreMissing
$self->testException(sub {backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, $strFileHash,
false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, false, undef, true, false, undef)},
ERROR_FILE_MISSING, "unable to open missing file '${strFileDb}.1' for read");
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")},
false, " $strRepoFile.1 section removed from manifest on undef host");
#---------------------------------------------------------------------------------------------------------------------------
# Restore the compressed file
executeTest('mv ' . "$strFileRepo.gz.SAVE $strFileRepo.gz");
# Yes prior checksum, yes compression, no page checksum, no extra, yes delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, $strBackupLabel,
true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_CHECKSUM && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize), true, 'db checksum and repo same - no copy file');
#---------------------------------------------------------------------------------------------------------------------------
# DB Checksum mismatch
storageTest()->remove("$strFileRepo", {bIgnoreMissing => true});
# Save the compressed file for later test
executeTest('mv ' . "$strFileRepo.gz $strFileRepo.gz.SAVE");
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash . "ff", false,
$strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db checksum mismatch - copy file');
#---------------------------------------------------------------------------------------------------------------------------
# DB file size mismatch
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false, $strBackupLabel, false,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db file size mismatch - copy file');
#---------------------------------------------------------------------------------------------------------------------------
# Repo mismatch
# Restore the compressed file as if non-compressed so checksum won't match
executeTest('cp ' . "$strFileRepo.gz.SAVE $strFileRepo");
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, $strBackupLabel, false,
cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_RECOPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'repo checksum mismatch - recopy file');
# Restore the compressed file
executeTest('mv ' . "$strFileRepo.gz.SAVE $strFileRepo.gz");
# Yes prior checksum, yes compression, no page checksum, no extra, no delta, no hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false,
$strBackupLabel, true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_RECOPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize), true, 'repo size mismatch - recopy file');
#---------------------------------------------------------------------------------------------------------------------------
# Has reference
# Set a reference in the manifest to ensure it is removed after backupManifestUpdate
$oBackupManifest->set(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REFERENCE, BOGUS);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REFERENCE,
BOGUS)}, true, "$strRepoFile reference section exists in manifest");
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, yes hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize + 1, $strFileHash, false,
$strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, true, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_COPY && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'db file size mismatch has reference - copy');
# Code path to ensure reference is removed
# Has reference - Code path to ensure reference is removed
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
@ -448,15 +292,15 @@ sub run
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent);
BACKUP_FILE_COPY,
16777216,
16777216,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
67117056,
167854,
0);
# Confirm reference to prior backup removed
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.",
@ -465,13 +309,6 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# BACKUP_FILE_NOOP
# Yes prior checksum, no compression, no page checksum, no extra, yes delta, yes hasReference
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false,
$strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, true, undef);
$self->testResult(($iResultCopyResult == BACKUP_FILE_NOOP && $strResultCopyChecksum eq $strFileHash &&
$lResultCopySize == $lFileSize), true, 'db file same has reference - noop');
# Calculate running counts
my $lSizeCurrentAfter = $lSizeCurrent + $lFileSize;
@ -489,27 +326,18 @@ sub run
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
BACKUP_FILE_NOOP,
16777216,
undef,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
83894272,
$lManifestSaveSize,
$lManifestSaveCurrent);
0);
$self->testResult(($lSizeCurrent ==$lSizeCurrentAfter && $lManifestSaveCurrent == $lManifestSaveCurrentAfter),
true, ' running counts updated');
#---------------------------------------------------------------------------------------------------------------------------
# Remove file from repo. No reference so should hard error since this means sometime between the building of the manifest
# for the aborted backup, the file went missing from the aborted backup dir.
storageTest()->remove("$strFileRepo", {bIgnoreMissing => true});
$self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash,
false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef)},
ERROR_FILE_MISSING, "unable to open missing file '${strFileRepo}' for read");
}
################################################################################################################################
@ -528,6 +356,7 @@ sub run
$lSizeCurrent = 0;
$lManifestSaveSize = $lFileSize * 2;
$lManifestSaveCurrent = 0;
$rResultExtra = undef;
$self->testResult(sub {backupManifestUpdate(
$oBackupManifest,
@ -588,7 +417,7 @@ sub run
$lManifestSaveCurrent)},
ERROR_ASSERT, "$strFileDb should have calculated page checksums");
$rResultExtra->{bValid} = false;
$rResultExtra->{valid} = false;
$self->testException(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
@ -607,9 +436,9 @@ sub run
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
ERROR_ASSERT, "bAlign flag should have been set for misaligned page");
ERROR_ASSERT, "align flag should have been set for misaligned page");
$rResultExtra->{bAlign} = true;
$rResultExtra->{align} = true;
$self->testException(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
@ -628,9 +457,9 @@ sub run
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
ERROR_ASSERT, "bAlign flag should have been set for misaligned page");
ERROR_ASSERT, "align flag should have been set for misaligned page");
$rResultExtra->{bAlign} = false;
$rResultExtra->{align} = false;
$self->testResult(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,

457
test/src/module/command/backupTest.c Normal file
View File

@ -0,0 +1,457 @@
/***********************************************************************************************************************************
Test Backup Command
***********************************************************************************************************************************/
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/io.h"
#include "storage/helper.h"
#include "storage/posix/storage.h"
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// Start a protocol server to test the protocol directly
Buffer *serverWrite = bufNew(8192);
IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite);
ioWriteOpen(serverWriteIo);
ProtocolServer *server = protocolServerNew(strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo);
bufUsedSet(serverWrite, 0);
const String *pgFile = strNew("testfile");
const String *missingFile = strNew("missing");
const String *backupLabel = strNew("20190718-155825F");
const String *backupPathFile = strNewFmt(STORAGE_REPO_BACKUP "/%s/%s", strPtr(backupLabel), strPtr(pgFile));
BackupFileResult result = {0};
VariantList *paramList = varLstNew();
// *****************************************************************************************************************************
if (testBegin("segmentNumber()"))
{
TEST_RESULT_UINT(segmentNumber(pgFile), 0, "No segment number");
TEST_RESULT_UINT(segmentNumber(strNewFmt("%s.123", strPtr(pgFile))), 123, "Segment number");
}
// *****************************************************************************************************************************
if (testBegin("backupFile(), backupProtocol"))
{
// Load Parameters
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAddZ(argList, "--repo1-retention-full=1");
strLstAddZ(argList, "backup");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Create the pg path
storagePathCreateP(storagePgWrite(), NULL, .mode = 0700);
// Pg file missing - ignoreMissing=true
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(
result,
backupFile(
missingFile, true, 0, NULL, false, 0, missingFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL),
"pg file missing, ignoreMissing=true, no delta");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy/repo size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, " skip file");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
// NULL, zero param values, ignoreMissing=true
varLstAdd(paramList, varNewStr(missingFile)); // pgFile
varLstAdd(paramList, varNewBool(true)); // pgFileIgnoreMissing
varLstAdd(paramList, varNewUInt64(0)); // pgFileSize
varLstAdd(paramList, NULL); // pgFileChecksum
varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2
varLstAdd(paramList, varNewStr(missingFile)); // repoFile
varLstAdd(paramList, varNewBool(false)); // repoFileHasReference
varLstAdd(paramList, varNewBool(false)); // repoFileCompress
varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel
varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
varLstAdd(paramList, varNewBool(false)); // delta
TEST_RESULT_BOOL(
backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - skip");
TEST_RESULT_STR(strPtr(strNewBuf(serverWrite)), "{\"out\":[3,0,0,null,null]}\n", " check result");
bufUsedSet(serverWrite, 0);
// Pg file missing - ignoreMissing=false
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT(
backupFile(
missingFile, false, 0, NULL, false, 0, missingFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL),
FileMissingError, "unable to open missing file '%s/pg/missing' for read", testPath());
// Create a pg file to backup
storagePutNP(storageNewWriteNP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile"));
// -------------------------------------------------------------------------------------------------------------------------
// No prior checksum, no compression, no pageChecksum, no delta, no hasReference
// With the expected backupCopyResultCopy, unset the storageFeatureCompress bit for the storageRepo for code coverage
uint64_t feature = storageRepo()->interface.feature;
((Storage *)storageRepo())->interface.feature = feature && ((1 << storageFeatureCompress) ^ 0xFFFFFFFFFFFFFFFF);
TEST_ASSIGN(
result,
backupFile(pgFile, false, 9, NULL, false, 0, pgFile, false, false, 1, backupLabel, false, cipherTypeNone, NULL),
"pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
((Storage *)storageRepo())->interface.feature = feature;
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy file to repo success");
TEST_RESULT_VOID(storageRemoveNP(storageRepoWrite(), backupPathFile), " remove repo file");
// -------------------------------------------------------------------------------------------------------------------------
// Test pagechecksum
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, NULL, true, 0xFFFFFFFFFFFFFFFF, pgFile, false, false, 1, backupLabel, false, cipherTypeNone,
NULL),
"file checksummed with pageChecksum enabled");
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile)),
true," copy file to repo success");
TEST_RESULT_PTR_NE(result.pageChecksumResult, NULL, " pageChecksumResult is set");
TEST_RESULT_BOOL(
varBool(kvGet(result.pageChecksumResult, VARSTRDEF("valid"))), false, " pageChecksumResult valid=false");
TEST_RESULT_VOID(storageRemoveNP(storageRepoWrite(), backupPathFile), " remove repo file");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
// pgFileSize, ignoreMissing=false, backupLabel, pgFileChecksumPage, pgFileChecksumPageLsnLimit
paramList = varLstNew();
varLstAdd(paramList, varNewStr(pgFile)); // pgFile
varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing
varLstAdd(paramList, varNewUInt64(9)); // pgFileSize
varLstAdd(paramList, NULL); // pgFileChecksum
varLstAdd(paramList, varNewBool(true)); // pgFileChecksumPage
varLstAdd(paramList, varNewUInt64(0xFFFFFFFF)); // pgFileChecksumPageLsnLimit 1
varLstAdd(paramList, varNewUInt64(0xFFFFFFFF)); // pgFileChecksumPageLsnLimit 2
varLstAdd(paramList, varNewStr(pgFile)); // repoFile
varLstAdd(paramList, varNewBool(false)); // repoFileHasReference
varLstAdd(paramList, varNewBool(false)); // repoFileCompress
varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel
varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
varLstAdd(paramList, varNewBool(false)); // delta
TEST_RESULT_BOOL(
backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - pageChecksum");
TEST_RESULT_STR(
strPtr(strNewBuf(serverWrite)),
"{\"out\":[1,9,9,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",{\"align\":false,\"valid\":false}]}\n",
" check result");
bufUsedSet(serverWrite, 0);
// -------------------------------------------------------------------------------------------------------------------------
// File exists in repo and db, checksum match, delta set, ignoreMissing false, hasReference - NOOP
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel,
true, cipherTypeNone, NULL),
"file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy size set");
TEST_RESULT_UINT(result.repoSize, 0, " repo size not set since already exists in repo");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultNoOp, " noop file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " noop");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
// pgFileChecksum, hasReference, delta
paramList = varLstNew();
varLstAdd(paramList, varNewStr(pgFile)); // pgFile
varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing
varLstAdd(paramList, varNewUInt64(9)); // pgFileSize
varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); // pgFileChecksum
varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2
varLstAdd(paramList, varNewStr(pgFile)); // repoFile
varLstAdd(paramList, varNewBool(true)); // repoFileHasReference
varLstAdd(paramList, varNewBool(false)); // repoFileCompress
varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel
varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
varLstAdd(paramList, varNewBool(true)); // delta
TEST_RESULT_BOOL(
backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - noop");
TEST_RESULT_STR(
strPtr(strNewBuf(serverWrite)), "{\"out\":[4,9,0,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n",
" check result");
bufUsedSet(serverWrite, 0);
// -------------------------------------------------------------------------------------------------------------------------
// File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, strNew("1234567890123456789012345678901234567890"), false, 0, pgFile, true, false, 1, backupLabel,
true, cipherTypeNone, NULL),
"file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy");
// -------------------------------------------------------------------------------------------------------------------------
// File exists in repo and db, pg checksum same, pg size different, delta set, ignoreMissing false, hasReference - COPY
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel,
true, cipherTypeNone, NULL),
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy");
// -------------------------------------------------------------------------------------------------------------------------
// File exists in repo and db, checksum not same in repo, delta set, ignoreMissing false, no hasReference - RECOPY
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageRepoWrite(), backupPathFile), BUFSTRDEF("adifferentfile")),
"create different file (size and checksum) with same name in repo");
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
backupLabel, true, cipherTypeNone, NULL),
" db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, " recopy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " recopy");
// -------------------------------------------------------------------------------------------------------------------------
// File exists in repo but missing from db, checksum same in repo, delta set, ignoreMissing true, no hasReference - SKIP
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageRepoWrite(), backupPathFile), BUFSTRDEF("adifferentfile")),
"create different file with same name in repo");
TEST_ASSIGN(
result,
backupFile(
missingFile, true, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
backupLabel, true, cipherTypeNone, NULL),
" file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy=repo=0 size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, " skip file");
TEST_RESULT_BOOL(
(result.copyChecksum == NULL && !storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " skip and remove file from repo");
// -------------------------------------------------------------------------------------------------------------------------
// No prior checksum, compression, no page checksum, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
backupFile(pgFile, false, 9, NULL, false, 0, pgFile, false, true, 3, backupLabel, false, cipherTypeNone, NULL),
"pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 29, " repo compress size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(backupLabel), strPtr(pgFile))) &&
result.pageChecksumResult == NULL),
true, " copy file to repo compress success");
// -------------------------------------------------------------------------------------------------------------------------
// Pg and repo file exist & match, prior checksum, compression, no page checksum, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, true, 3, backupLabel,
false, cipherTypeNone, NULL),
"pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 29, " repo compress size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultChecksum, " checksum file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(backupLabel), strPtr(pgFile))) &&
result.pageChecksumResult == NULL),
true, " compressed repo file matches");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
// compression
paramList = varLstNew();
varLstAdd(paramList, varNewStr(pgFile)); // pgFile
varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing
varLstAdd(paramList, varNewUInt64(9)); // pgFileSize
varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); // pgFileChecksum
varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2
varLstAdd(paramList, varNewStr(pgFile)); // repoFile
varLstAdd(paramList, varNewBool(false)); // repoFileHasReference
varLstAdd(paramList, varNewBool(true)); // repoFileCompress
varLstAdd(paramList, varNewUInt(3)); // repoFileCompressLevel
varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
varLstAdd(paramList, varNewBool(false)); // delta
TEST_RESULT_BOOL(
backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - copy, compress");
TEST_RESULT_STR(
strPtr(strNewBuf(serverWrite)), "{\"out\":[0,9,29,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n",
" check result");
bufUsedSet(serverWrite, 0);
// -------------------------------------------------------------------------------------------------------------------------
// Create a zero sized file - checksum will be set but in backupManifestUpdate it will not be copied
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("zerofile")), BUFSTRDEF(""));
// No prior checksum, no compression, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
backupFile(
strNew("zerofile"), false, 0, NULL, false, 0, strNew("zerofile"), false, false, 1, backupLabel, false,
cipherTypeNone, NULL),
"zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, " copy=repo=pgFile size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_PTR_NE(result.copyChecksum, NULL, " checksum set");
TEST_RESULT_BOOL(
(storageExistsNP(storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/zerofile", strPtr(backupLabel))) &&
result.pageChecksumResult == NULL),
true, " copy zero file to repo success");
// Check invalid protocol function
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL(backupProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
}
// *****************************************************************************************************************************
if (testBegin("backupFile() - encrypt"))
{
// Load Parameters
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAddZ(argList, "--repo1-retention-full=1");
strLstAddZ(argList, "--repo1-cipher-type=aes-256-cbc");
strLstAddZ(argList, "backup");
setenv("PGBACKREST_REPO1_CIPHER_PASS", "12345678", true);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
unsetenv("PGBACKREST_REPO1_CIPHER_PASS");
// Create the pg path
storagePathCreateP(storagePgWrite(), NULL, .mode = 0700);
// Create a pg file to backup
storagePutNP(storageNewWriteNP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile"));
// -------------------------------------------------------------------------------------------------------------------------
// No prior checksum, no compression, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, NULL, false, 0, pgFile, false, false, 1, backupLabel, false, cipherTypeAes256Cbc,
strNew("12345678")),
"pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy size set");
TEST_RESULT_UINT(result.repoSize, 32, " repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy file to encrypted repo success");
// -------------------------------------------------------------------------------------------------------------------------
// Delta but pgMatch false (pg File size different), prior checksum, no compression, no pageChecksum, delta, no hasReference
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
backupLabel, true, cipherTypeAes256Cbc, strNew("12345678")),
"pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy size set");
TEST_RESULT_UINT(result.repoSize, 32, " repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy file (size missmatch) to encrypted repo success");
// -------------------------------------------------------------------------------------------------------------------------
// Check repo with cipher filter.
// pg/repo file size same but checksum different, prior checksum, no compression, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 9, strNew("1234567890123456789012345678901234567890"), false, 0, pgFile, false, false, 0,
backupLabel, false, cipherTypeAes256Cbc, strNew("12345678")),
"pg and repo file exists, repo checksum no match, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy size set");
TEST_RESULT_UINT(result.repoSize, 32, " repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, " recopy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
storageExistsNP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " recopy file to encrypted repo success");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
// cipherType, cipherPass
paramList = varLstNew();
varLstAdd(paramList, varNewStr(pgFile)); // pgFile
varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing
varLstAdd(paramList, varNewUInt64(9)); // pgFileSize
varLstAdd(paramList, varNewStrZ("1234567890123456789012345678901234567890")); // pgFileChecksum
varLstAdd(paramList, varNewBool(false)); // pgFileChecksumPage
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 1
varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit 2
varLstAdd(paramList, varNewStr(pgFile)); // repoFile
varLstAdd(paramList, varNewBool(false)); // repoFileHasReference
varLstAdd(paramList, varNewBool(false)); // repoFileCompress
varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel
varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel
varLstAdd(paramList, varNewBool(false)); // delta
varLstAdd(paramList, varNewStrZ("12345678")); // cipherPass
TEST_RESULT_BOOL(
backupProtocol(PROTOCOL_COMMAND_BACKUP_FILE_STR, paramList, server), true, "protocol backup file - recopy, encrypt");
TEST_RESULT_STR(
strPtr(strNewBuf(serverWrite)), "{\"out\":[2,9,32,\"9bc8ab2dda60ef4beed07d1e19ce0676d5edde67\",null]}\n",
" check result");
bufUsedSet(serverWrite, 0);
}
FUNCTION_HARNESS_RESULT_VOID();
}