mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-03-03 14:52:21 +02:00
Add Perl interface to C storage layer.
Maintaining the storage layer/drivers in two languages is burdensome. Since the integration tests require the Perl storage layer/drivers we'll need them even after the core code is migrated to C. Create an interface layer so the Perl code can be removed and new storage drivers/features introduced without adding Perl equivalents. The goal is to move the integration tests to C so this interface will eventually be removed. That being the case, the interface was designed for maximum compatibility to ease the transition. The result looks a bit hacky but we'll improve it as needed until it can be retired.
This commit is contained in:
David Steele
parent
bd6c0941e9
commit
4815752ccc
@ -13,6 +13,17 @@
|
||||
|
||||
<release-list>
|
||||
<release date="XXXX-XX-XX" version="2.16dev" title="UNDER DEVELOPMENT">
|
||||
<release-core-list>
|
||||
<release-development-list>
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-reviewer id="cynthia.shang"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Add Perl interface to C storage layer.</p>
|
||||
</release-item>
|
||||
</release-development-list>
|
||||
</release-core-list>
|
||||
</release>
|
||||
|
||||
<release date="2019-06-25" version="2.15" title="C Implementation of Expire">
|
||||
|
||||
@ -584,20 +584,6 @@
|
||||
|
||||
<!-- ======================================================================================================================= -->
|
||||
<block-define id="s3-setup">
|
||||
<execute-list host="{[s3-setup-host]}">
|
||||
<title>Install packages required for S3-compatible object store support</title>
|
||||
|
||||
<execute if="{[os-type-is-debian]}" user="root" pre="y">
|
||||
<exe-cmd>apt-get install libio-socket-ssl-perl libxml-libxml-perl</exe-cmd>
|
||||
<exe-cmd-extra>-y 2>&1</exe-cmd-extra>
|
||||
</execute>
|
||||
|
||||
<execute if="{[os-type-is-centos]}" user="root" pre="y">
|
||||
<exe-cmd>yum install perl-XML-LibXML perl-IO-Socket-SSL</exe-cmd>
|
||||
<exe-cmd-extra>-y 2>&1</exe-cmd-extra>
|
||||
</execute>
|
||||
</execute-list>
|
||||
|
||||
<p><backrest/> supports locating repositories in <proper>S3-compatible</proper> object stores. The bucket used to store the repository must be created in advance &mdash; <backrest/> will not do it automatically. The repository can be located in the bucket root (<path>/</path>) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.</p>
|
||||
|
||||
<execute-list if="'{[s3-local]}' eq 'y'" host="{[s3-setup-host]}" show="n">
|
||||
@ -627,7 +613,6 @@
|
||||
<backrest-config-option section="global" key="repo1-s3-bucket">{[s3-bucket]}</backrest-config-option>
|
||||
<backrest-config-option section="global" key="repo1-s3-endpoint">{[s3-endpoint]}</backrest-config-option>
|
||||
<backrest-config-option section="global" key="repo1-s3-region">{[s3-region]}</backrest-config-option>
|
||||
<backrest-config-option section="global" key="repo1-s3-ca-file" if="{[os-type-is-centos]}">/etc/pki/tls/certs/ca-bundle.crt</backrest-config-option>
|
||||
|
||||
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
|
||||
</backrest-config>
|
||||
|
||||
@ -21,9 +21,6 @@ use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
####################################################################################################################################
|
||||
|
||||
@ -27,7 +27,6 @@ use pgBackRest::InfoCommon;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
####################################################################################################################################
|
||||
@ -409,9 +408,6 @@ sub reconstruct
|
||||
# Get the db-system-id from the WAL file depending on the version of postgres
|
||||
my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;
|
||||
|
||||
# Read first 8k of WAL segment
|
||||
my $tBlock;
|
||||
|
||||
# Error if the file encryption setting is not valid for the repo
|
||||
if (!storageRepo()->encryptionValid(storageRepo()->encrypted($strArchiveFilePath)))
|
||||
{
|
||||
@ -423,10 +419,12 @@ sub reconstruct
|
||||
my $oFileIo = storageRepo()->openRead(
|
||||
$strArchiveFilePath,
|
||||
{rhyFilter => $strArchiveFile =~ ('\.' . COMPRESS_EXT . '$') ?
|
||||
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef,
|
||||
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}] : undef,
|
||||
strCipherPass => $self->cipherPassSub()});
|
||||
$oFileIo->open();
|
||||
|
||||
$oFileIo->read(\$tBlock, 512, true);
|
||||
my $tBlock;
|
||||
$oFileIo->read(\$tBlock, 512);
|
||||
$oFileIo->close();
|
||||
|
||||
# Get the required data from the file that was pulled into scalar $tBlock
|
||||
|
||||
@ -30,8 +30,6 @@ use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Common::Io::Handle;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Helper;
|
||||
use pgBackRest::Version;
|
||||
|
||||
@ -229,7 +227,7 @@ sub resumeClean
|
||||
if ($cType eq 'd')
|
||||
{
|
||||
logDebugMisc($strOperation, "remove path ${strName}");
|
||||
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}", {bRecurse => true});
|
||||
$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}", {bRecurse => true});
|
||||
}
|
||||
# Else add the file/link to be deleted later
|
||||
else
|
||||
@ -323,7 +321,7 @@ sub processManifest
|
||||
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strPath}", {bIgnoreExists => true});
|
||||
}
|
||||
|
||||
if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))
|
||||
if (storageRepo()->capability(STORAGE_CAPABILITY_LINK))
|
||||
{
|
||||
for my $strTarget ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_TARGET))
|
||||
{
|
||||
@ -745,7 +743,7 @@ sub process
|
||||
&log(WARN, "aborted backup ${strAbortedBackup} cannot be resumed: ${strReason}");
|
||||
&log(TEST, TEST_BACKUP_NORESUME);
|
||||
|
||||
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strAbortedBackup}", {bRecurse => true});
|
||||
$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . "/${strAbortedBackup}", {bRecurse => true});
|
||||
undef($oAbortedManifest);
|
||||
}
|
||||
|
||||
@ -979,7 +977,9 @@ sub process
|
||||
# Add compression filter
|
||||
if ($bCompress)
|
||||
{
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP});
|
||||
push(
|
||||
@{$rhyFilter},
|
||||
{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, cfgOption(CFGOPT_COMPRESS_LEVEL)]});
|
||||
}
|
||||
|
||||
# If the backups are encrypted, then the passphrase for the backup set from the manifest file is required to access
|
||||
@ -1064,7 +1064,7 @@ sub process
|
||||
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel);
|
||||
|
||||
# Sync backup path if supported
|
||||
if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
{
|
||||
# Sync all paths in the backup
|
||||
$oStorageRepo->pathSync(STORAGE_REPO_BACKUP . "/${strBackupLabel}");
|
||||
@ -1096,12 +1096,12 @@ sub process
|
||||
{'strCipherPass' => $strCipherPassManifest}),
|
||||
$oStorageRepo->openWrite(
|
||||
"${strHistoryPath}/${strBackupLabel}.manifest." . COMPRESS_EXT,
|
||||
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP}],
|
||||
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, 9]}],
|
||||
bPathCreate => true, bAtomic => true,
|
||||
strCipherPass => defined($strCipherPassManifest) ? $strCipherPassManifest : undef}));
|
||||
|
||||
# Sync history path if supported
|
||||
if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
{
|
||||
$oStorageRepo->pathSync(STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY);
|
||||
$oStorageRepo->pathSync($strHistoryPath);
|
||||
@ -1110,7 +1110,7 @@ sub process
|
||||
# Create a link to the most recent backup
|
||||
$oStorageRepo->remove(STORAGE_REPO_BACKUP . qw(/) . LINK_LATEST);
|
||||
|
||||
if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))
|
||||
if (storageRepo()->capability(STORAGE_CAPABILITY_LINK))
|
||||
{
|
||||
$oStorageRepo->linkCreate(
|
||||
STORAGE_REPO_BACKUP . "/${strBackupLabel}", STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST, {bRelative => true});
|
||||
@ -1120,7 +1120,7 @@ sub process
|
||||
$oBackupInfo->add($oBackupManifest);
|
||||
|
||||
# Sync backup root path if supported
|
||||
if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))
|
||||
{
|
||||
$oStorageRepo->pathSync(STORAGE_REPO_BACKUP);
|
||||
}
|
||||
|
||||
@ -12,17 +12,15 @@ use Exporter qw(import);
|
||||
use File::Basename qw(dirname);
|
||||
use Storable qw(dclone);
|
||||
|
||||
use pgBackRest::Backup::Filter::PageChecksum;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Handle;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
####################################################################################################################################
|
||||
@ -143,7 +141,7 @@ sub backupFile
|
||||
|
||||
if ($bCompress)
|
||||
{
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]});
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]});
|
||||
}
|
||||
|
||||
# Get the checksum
|
||||
@ -163,8 +161,8 @@ sub backupFile
|
||||
# Copy the file
|
||||
if ($bCopy)
|
||||
{
|
||||
# Add sha filter
|
||||
my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];
|
||||
# Add size and sha filters
|
||||
my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}];
|
||||
|
||||
# Add page checksum filter
|
||||
if ($bChecksumPage)
|
||||
@ -174,29 +172,44 @@ sub backupFile
|
||||
|
||||
push(
|
||||
@{$rhyFilter},
|
||||
{strClass => BACKUP_FILTER_PAGECHECKSUM,
|
||||
{strClass => "pgBackRest::Backup::Filter::PageChecksum",
|
||||
rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});
|
||||
};
|
||||
|
||||
# Add compression
|
||||
if ($bCompress)
|
||||
{
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => $iCompressLevel}]});
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]});
|
||||
}
|
||||
# Else add protocol compression if the destination is not compressed and there is no encryption
|
||||
elsif (!defined($strCipherPass))
|
||||
{
|
||||
push(
|
||||
@{$rhyFilter},
|
||||
{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]});
|
||||
}
|
||||
|
||||
# Open the file
|
||||
# Open the source file
|
||||
my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing});
|
||||
|
||||
# If source file exists
|
||||
if (defined($oSourceFileIo))
|
||||
# Open the destination file
|
||||
$rhyFilter = undef;
|
||||
|
||||
# Add protocol decompression if the destination is not compressed and there is no encryption
|
||||
if (!$bCompress && !defined($strCipherPass))
|
||||
{
|
||||
my $oDestinationFileIo = $oStorageRepo->openWrite(
|
||||
STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
|
||||
{bPathCreate => true, bProtocolCompress => !$bCompress, strCipherPass => $strCipherPass});
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]});
|
||||
}
|
||||
|
||||
# Copy the file
|
||||
$oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo);
|
||||
my $oDestinationFileIo = $oStorageRepo->openWrite(
|
||||
STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}",
|
||||
{bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass});
|
||||
|
||||
$oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef);
|
||||
|
||||
# Copy the file
|
||||
if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo))
|
||||
{
|
||||
# Get sha checksum and size
|
||||
$strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);
|
||||
$lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);
|
||||
@ -208,7 +221,17 @@ sub backupFile
|
||||
}
|
||||
|
||||
# Get results of page checksum validation
|
||||
$rExtra = $bChecksumPage ? $oSourceFileIo->result(BACKUP_FILTER_PAGECHECKSUM) : undef;
|
||||
if ($bChecksumPage)
|
||||
{
|
||||
my $rExtraRaw = $oSourceFileIo->result("pgBackRest::Backup::Filter::PageChecksum");
|
||||
|
||||
$rExtra =
|
||||
{
|
||||
bValid => $rExtraRaw->{valid} ? true : false,
|
||||
bAlign => $rExtraRaw->{align} ? true : false,
|
||||
iyPageError => $rExtraRaw->{error},
|
||||
};
|
||||
}
|
||||
}
|
||||
# Else if source file is missing the database removed it
|
||||
else
|
||||
@ -223,21 +246,21 @@ sub backupFile
|
||||
#
|
||||
# If the file was checksummed then get the size in all cases since we don't already have it.
|
||||
if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) &&
|
||||
$oStorageRepo->driver()->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||
|
||||
$oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||
|
||||
$iCopyResult == BACKUP_FILE_CHECKSUM)
|
||||
{
|
||||
$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->size();
|
||||
$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->{size};
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'iCopyResult', value => $iCopyResult, trace => true},
|
||||
{name => 'lCopySize', value => $lCopySize, trace => true},
|
||||
{name => 'lRepoSize', value => $lRepoSize, trace => true},
|
||||
{name => 'strCopyChecksum', value => $strCopyChecksum, trace => true},
|
||||
{name => 'rExtra', value => $rExtra, trace => true},
|
||||
{name => 'iCopyResult', value => $iCopyResult},
|
||||
{name => 'lCopySize', value => $lCopySize},
|
||||
{name => 'lRepoSize', value => $lRepoSize},
|
||||
{name => 'strCopyChecksum', value => $strCopyChecksum},
|
||||
{name => 'rExtra', value => $rExtra},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -1,168 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Backup Page Checksum Filter
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Backup::Filter::PageChecksum;
|
||||
use parent 'pgBackRest::Common::Io::Filter';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::DbVersion qw(PG_PAGE_SIZE);
|
||||
use pgBackRest::LibC qw(:checksum);
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant BACKUP_FILTER_PAGECHECKSUM => __PACKAGE__;
|
||||
push @EXPORT, qw(BACKUP_FILTER_PAGECHECKSUM);
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oParent,
|
||||
$iSegmentNo,
|
||||
$iWalId,
|
||||
$iWalOffset,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oParent', trace => true},
|
||||
{name => 'iSegmentNo', trace => true},
|
||||
{name => 'iWalId', trace => true},
|
||||
{name => 'iWalOffset', trace => true},
|
||||
);
|
||||
|
||||
# Bless with new class
|
||||
my $self = $class->SUPER::new($oParent);
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{iSegmentNo} = $iSegmentNo;
|
||||
$self->{iWalId} = $iWalId;
|
||||
$self->{iWalOffset} = $iWalOffset;
|
||||
|
||||
# Create the result object
|
||||
$self->{hResult}{bValid} = true;
|
||||
$self->{hResult}{bAlign} = true;
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# read - validate page checksums
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
my $iSize = shift;
|
||||
|
||||
# Call the io method
|
||||
my $iActualSize = $self->parent()->read($rtBuffer, $iSize);
|
||||
|
||||
# Validate page checksums for the read block
|
||||
if ($iActualSize > 0)
|
||||
{
|
||||
# If the buffer is not divisible by 0 then it's not valid
|
||||
if (!$self->{hResult}{bAlign} || ($iActualSize % PG_PAGE_SIZE != 0))
|
||||
{
|
||||
if (!$self->{hResult}{bAlign})
|
||||
{
|
||||
confess &log(ASSERT, "should not be possible to see two misaligned blocks in a row");
|
||||
}
|
||||
|
||||
$self->{hResult}{bValid} = false;
|
||||
$self->{hResult}{bAlign} = false;
|
||||
delete($self->{hResult}{iyPageError});
|
||||
}
|
||||
else
|
||||
{
|
||||
# Calculate offset to the first block in the buffer
|
||||
my $iBlockOffset = int(($self->size() - $iActualSize) / PG_PAGE_SIZE) + ($self->{iSegmentNo} * 131072);
|
||||
|
||||
if (!pageChecksumBufferTest(
|
||||
$$rtBuffer, $iActualSize, $iBlockOffset, PG_PAGE_SIZE, $self->{iWalId},
|
||||
$self->{iWalOffset}))
|
||||
{
|
||||
$self->{hResult}{bValid} = false;
|
||||
|
||||
# Now figure out exactly where the errors occurred. It would be more efficient if the checksum function returned an
|
||||
# array, but we're hoping there won't be that many errors to scan so this should work fine.
|
||||
for (my $iBlockNo = 0; $iBlockNo < int($iActualSize / PG_PAGE_SIZE); $iBlockNo++)
|
||||
{
|
||||
my $iBlockNoStart = $iBlockOffset + $iBlockNo;
|
||||
|
||||
if (!pageChecksumTest(
|
||||
substr($$rtBuffer, $iBlockNo * PG_PAGE_SIZE, PG_PAGE_SIZE), $iBlockNoStart, PG_PAGE_SIZE,
|
||||
$self->{iWalId}, $self->{iWalOffset}))
|
||||
{
|
||||
my $iLastIdx = defined($self->{hResult}{iyPageError}) ? @{$self->{hResult}{iyPageError}} - 1 : 0;
|
||||
my $iyLast = defined($self->{hResult}{iyPageError}) ? $self->{hResult}{iyPageError}[$iLastIdx] : undef;
|
||||
|
||||
if (!defined($iyLast) || (!ref($iyLast) && $iyLast != $iBlockNoStart - 1) ||
|
||||
(ref($iyLast) && $iyLast->[1] != $iBlockNoStart - 1))
|
||||
{
|
||||
push(@{$self->{hResult}{iyPageError}}, $iBlockNoStart);
|
||||
}
|
||||
elsif (!ref($iyLast))
|
||||
{
|
||||
$self->{hResult}{iyPageError}[$iLastIdx] = undef;
|
||||
push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iyLast);
|
||||
push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iBlockNoStart);
|
||||
}
|
||||
else
|
||||
{
|
||||
$self->{hResult}{iyPageError}[$iLastIdx][1] = $iBlockNoStart;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Return the actual size read
|
||||
return $iActualSize;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close and set the result
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->{hResult}))
|
||||
{
|
||||
# Set result
|
||||
$self->resultSet(BACKUP_FILTER_PAGECHECKSUM, $self->{hResult});
|
||||
|
||||
# Delete the sha object
|
||||
undef($self->{hResult});
|
||||
|
||||
# Close io
|
||||
return $self->parent()->close();
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,364 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# HTTP Client
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Common::Http::Client;
|
||||
use parent 'pgBackRest::Common::Io::Buffered';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use IO::Socket::SSL;
|
||||
use Socket qw(SOL_SOCKET SO_KEEPALIVE);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Buffered;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Common::Xml;
|
||||
use pgBackRest::Common::Http::Common;
|
||||
|
||||
####################################################################################################################################
|
||||
# Constants
|
||||
####################################################################################################################################
|
||||
use constant HTTP_VERB_GET => 'GET';
|
||||
push @EXPORT, qw(HTTP_VERB_GET);
|
||||
use constant HTTP_VERB_POST => 'POST';
|
||||
push @EXPORT, qw(HTTP_VERB_POST);
|
||||
use constant HTTP_VERB_PUT => 'PUT';
|
||||
push @EXPORT, qw(HTTP_VERB_PUT);
|
||||
|
||||
use constant HTTP_HEADER_CONTENT_LENGTH => 'content-length';
|
||||
push @EXPORT, qw(HTTP_HEADER_CONTENT_LENGTH);
|
||||
use constant HTTP_HEADER_TRANSFER_ENCODING => 'transfer-encoding';
|
||||
push @EXPORT, qw(HTTP_HEADER_TRANSFER_ENCODING);
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strHost,
|
||||
$strVerb,
|
||||
$iPort,
|
||||
$strUri,
|
||||
$hQuery,
|
||||
$hRequestHeader,
|
||||
$rstrRequestBody,
|
||||
$bResponseBodyPrefetch,
|
||||
$iProtocolTimeout,
|
||||
$iTryTotal,
|
||||
$lBufferMax,
|
||||
$bVerifySsl,
|
||||
$strCaPath,
|
||||
$strCaFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'strHost', trace => true},
|
||||
{name => 'strVerb', trace => true},
|
||||
{name => 'iPort', optional => true, default => 443, trace => true},
|
||||
{name => 'strUri', optional => true, default => qw(/), trace => true},
|
||||
{name => 'hQuery', optional => true, trace => true},
|
||||
{name => 'hRequestHeader', optional => true, trace => true},
|
||||
{name => 'rstrRequestBody', optional => true, trace => true},
|
||||
{name => 'bResponseBodyPrefetch', optional => true, default => false, trace => true},
|
||||
{name => 'iProtocolTimeout', optional => true, default => 300, trace => true},
|
||||
{name => 'iTryTotal', optional => true, default => 3, trace => true},
|
||||
{name => 'lBufferMax', optional => true, default => 32768, trace => true},
|
||||
{name => 'bVerifySsl', optional => true, default => true, trace => true},
|
||||
{name => 'strCaPath', optional => true, trace => true},
|
||||
{name => 'strCaFile', optional => true, trace => true},
|
||||
);
|
||||
|
||||
# Retry as many times as requested
|
||||
my $self;
|
||||
my $iTry = 1;
|
||||
my $bRetry;
|
||||
|
||||
do
|
||||
{
|
||||
# Disable logging if a failure will be retried
|
||||
logDisable() if $iTry < $iTryTotal;
|
||||
$bRetry = false;
|
||||
|
||||
eval
|
||||
{
|
||||
# Connect to the server
|
||||
my $oSocket;
|
||||
|
||||
if (eval{require IO::Socket::IP})
|
||||
{
|
||||
$oSocket = IO::Socket::IP->new(PeerHost => $strHost, PeerPort => $iPort)
|
||||
or confess &log(ERROR, "unable to create socket: $@", ERROR_HOST_CONNECT);
|
||||
}
|
||||
else
|
||||
{
|
||||
require IO::Socket::INET;
|
||||
|
||||
$oSocket = IO::Socket::INET->new(PeerHost => $strHost, PeerPort => $iPort)
|
||||
or confess &log(ERROR, "unable to create socket: $@", ERROR_HOST_CONNECT);
|
||||
}
|
||||
|
||||
setsockopt($oSocket, SOL_SOCKET,SO_KEEPALIVE, 1)
|
||||
or confess &log(ERROR, "unable to set socket keepalive: $@", ERROR_HOST_CONNECT);
|
||||
|
||||
eval
|
||||
{
|
||||
IO::Socket::SSL->start_SSL(
|
||||
$oSocket, SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, SSL_ca_path => $strCaPath,
|
||||
SSL_ca_file => $strCaFile);
|
||||
}
|
||||
or do
|
||||
{
|
||||
logErrorResult(
|
||||
ERROR_HOST_CONNECT, coalesce(length($!) == 0 ? undef : $!, $SSL_ERROR), length($!) > 0 ? $SSL_ERROR : undef);
|
||||
};
|
||||
|
||||
# Bless with new class
|
||||
$self = $class->SUPER::new(
|
||||
new pgBackRest::Common::Io::Handle('httpClient', $oSocket, $oSocket), $iProtocolTimeout, $lBufferMax);
|
||||
bless $self, $class;
|
||||
|
||||
# Store socket
|
||||
$self->{oSocket} = $oSocket;
|
||||
|
||||
# Generate the query string
|
||||
my $strQuery = httpQuery($hQuery);
|
||||
|
||||
# Construct the request headers
|
||||
$self->{strRequestHeader} = "${strVerb} " . httpUriEncode($strUri, true) . "?${strQuery} HTTP/1.1" . "\r\n";
|
||||
|
||||
foreach my $strHeader (sort(keys(%{$hRequestHeader})))
|
||||
{
|
||||
$self->{strRequestHeader} .= "${strHeader}: $hRequestHeader->{$strHeader}\r\n";
|
||||
}
|
||||
|
||||
$self->{strRequestHeader} .= "\r\n";
|
||||
|
||||
# Write request headers
|
||||
$self->write(\$self->{strRequestHeader});
|
||||
|
||||
# Write content
|
||||
if (defined($rstrRequestBody))
|
||||
{
|
||||
my $iTotalSize = length($$rstrRequestBody);
|
||||
my $iTotalSent = 0;
|
||||
|
||||
# Write the request body in buffer-sized chunks
|
||||
do
|
||||
{
|
||||
my $strBufferWrite = substr($$rstrRequestBody, $iTotalSent, $lBufferMax);
|
||||
$iTotalSent += $self->write(\$strBufferWrite);
|
||||
} while ($iTotalSent < $iTotalSize);
|
||||
}
|
||||
|
||||
# Read response code
|
||||
($self->{strResponseProtocol}, $self->{iResponseCode}, $self->{strResponseMessage}) =
|
||||
split(' ', trim($self->readLine()));
|
||||
|
||||
# Read the response headers
|
||||
$self->{iContentLength} = 0;
|
||||
$self->{strResponseHeader} = '';
|
||||
my $strHeader = trim($self->readLine());
|
||||
|
||||
while ($strHeader ne '')
|
||||
{
|
||||
# Validate header
|
||||
$self->{strResponseHeader} .= "${strHeader}\n";
|
||||
|
||||
my $iColonPos = index($strHeader, ':');
|
||||
|
||||
if ($iColonPos == -1)
|
||||
{
|
||||
confess &log(ERROR, "http header '${strHeader}' requires colon separator", ERROR_PROTOCOL);
|
||||
}
|
||||
|
||||
# Parse header
|
||||
my $strHeaderKey = lc(substr($strHeader, 0, $iColonPos));
|
||||
my $strHeaderValue = trim(substr($strHeader, $iColonPos + 1));
|
||||
|
||||
# Store the header
|
||||
$self->{hResponseHeader}{$strHeaderKey} = $strHeaderValue;
|
||||
|
||||
# Process content length
|
||||
if ($strHeaderKey eq HTTP_HEADER_CONTENT_LENGTH)
|
||||
{
|
||||
$self->{iContentLength} = $strHeaderValue + 0;
|
||||
$self->{iContentRemaining} = $self->{iContentLength};
|
||||
}
|
||||
# Process transfer encoding (only chunked is supported)
|
||||
elsif ($strHeaderKey eq HTTP_HEADER_TRANSFER_ENCODING)
|
||||
{
|
||||
if ($strHeaderValue eq 'chunked')
|
||||
{
|
||||
$self->{iContentLength} = -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
confess &log(ERROR, "invalid value '${strHeaderValue} for http header '${strHeaderKey}'", ERROR_PROTOCOL);
|
||||
}
|
||||
}
|
||||
|
||||
# Read next header
|
||||
$strHeader = trim($self->readLine());
|
||||
}
|
||||
|
||||
# Prefetch response - mostly useful when the response is known to be short
|
||||
if ($bResponseBodyPrefetch)
|
||||
{
|
||||
$self->{strResponseBody} = $self->responseBody();
|
||||
}
|
||||
|
||||
# Enable logging if a failure will be retried
|
||||
logEnable() if $iTry < $iTryTotal;
|
||||
return 1;
|
||||
}
|
||||
or do
|
||||
{
|
||||
# Enable logging if a failure will be retried
|
||||
logEnable() if $iTry < $iTryTotal;
|
||||
|
||||
# If tries reaches total allowed then error
|
||||
if ($iTry == $iTryTotal)
|
||||
{
|
||||
confess $EVAL_ERROR;
|
||||
}
|
||||
|
||||
# Try again
|
||||
$iTry++;
|
||||
$bRetry = true;
|
||||
};
|
||||
}
|
||||
while ($bRetry);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# read - read content from http stream
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
my $iRequestSize = shift;
|
||||
|
||||
# Make sure request size is not larger than what remains to be read
|
||||
$iRequestSize = $iRequestSize < $self->{iContentRemaining} ? $iRequestSize : $self->{iContentRemaining};
|
||||
$self->{iContentRemaining} -= $iRequestSize;
|
||||
|
||||
my $iActualSize = $self->SUPER::read($rtBuffer, $iRequestSize, true);
|
||||
|
||||
# Set eof if there is nothing left to read
|
||||
if ($self->{iContentRemaining} == 0)
|
||||
{
|
||||
$self->SUPER::eofSet(true);
|
||||
}
|
||||
|
||||
return $iActualSize;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close/DESTROY - close the HTTP connection
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Only close if the socket is open
|
||||
if (defined($self->{oSocket}))
|
||||
{
|
||||
$self->{oSocket}->close();
|
||||
undef($self->{oSocket});
|
||||
}
|
||||
}
|
||||
|
||||
sub DESTROY {shift->close()}
|
||||
|
||||
####################################################################################################################################
|
||||
# responseBody - return the entire body of the response in a buffer
|
||||
####################################################################################################################################
|
||||
sub responseBody
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->responseBody'
|
||||
);
|
||||
|
||||
# Return prefetched response body if it exists
|
||||
return $self->{strResponseBody} if exists($self->{strResponseBody});
|
||||
|
||||
# Fetch response body if content length is not 0
|
||||
my $strResponseBody = undef;
|
||||
|
||||
if ($self->{iContentLength} != 0)
|
||||
{
|
||||
# Transfer encoding is chunked
|
||||
if ($self->{iContentLength} == -1)
|
||||
{
|
||||
while (1)
|
||||
{
|
||||
# Read chunk length
|
||||
my $strChunkLength = trim($self->readLine());
|
||||
my $iChunkLength = hex($strChunkLength);
|
||||
|
||||
# Exit if chunk length is 0
|
||||
last if ($iChunkLength == 0);
|
||||
|
||||
# Read the chunk and consume the terminating LF
|
||||
$self->SUPER::read(\$strResponseBody, $iChunkLength, true);
|
||||
$self->readLine();
|
||||
};
|
||||
}
|
||||
# Else content length is known
|
||||
else
|
||||
{
|
||||
$self->SUPER::read(\$strResponseBody, $self->{iContentLength}, true);
|
||||
}
|
||||
|
||||
$self->close();
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'rstrResponseBody', value => \$strResponseBody, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Properties.
|
||||
####################################################################################################################################
|
||||
sub contentLength {shift->{iContentLength}} # Content length if available (-1 means not known yet)
|
||||
sub requestHeaderText {trim(shift->{strRequestHeader})}
|
||||
sub responseCode {shift->{iResponseCode}}
|
||||
sub responseHeader {shift->{hResponseHeader}}
|
||||
sub responseHeaderText {trim(shift->{strResponseHeader})}
|
||||
sub responseMessage {shift->{strResponseMessage}}
|
||||
sub responseProtocol {shift->{strResponseProtocol}}
|
||||
|
||||
1;
|
||||
@ -1,107 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# HTTP Common
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Common::Http::Common;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# httpQuery - encode an HTTP query from a hash
|
||||
####################################################################################################################################
|
||||
sub httpQuery
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$hQuery,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::httpQuery', \@_,
|
||||
{name => 'hQuery', required => false, trace => true},
|
||||
);
|
||||
|
||||
# Generate the query string
|
||||
my $strQuery = '';
|
||||
|
||||
# If a hash (the normal case)
|
||||
if (ref($hQuery))
|
||||
{
|
||||
foreach my $strParam (sort(keys(%{$hQuery})))
|
||||
{
|
||||
# Parameters may not be defined - this is OK
|
||||
if (defined($hQuery->{$strParam}))
|
||||
{
|
||||
$strQuery .= ($strQuery eq '' ? '' : '&') . $strParam . '=' . httpUriEncode($hQuery->{$strParam});
|
||||
}
|
||||
}
|
||||
}
|
||||
# Else query string was passed directly as a scalar
|
||||
elsif (defined($hQuery))
|
||||
{
|
||||
$strQuery = $hQuery;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strQuery', value => $strQuery, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(httpQuery);
|
||||
|
||||
####################################################################################################################################
|
||||
# httpUriEncode - encode query values to conform with URI specs
|
||||
####################################################################################################################################
|
||||
sub httpUriEncode
|
||||
{
|
||||
my $strString = shift;
|
||||
my $bPath = shift;
|
||||
|
||||
# Only encode if source string is defined
|
||||
my $strEncodedString;
|
||||
|
||||
if (defined($strString))
|
||||
{
|
||||
# Iterate all characters in the string
|
||||
for (my $iIndex = 0; $iIndex < length($strString); $iIndex++)
|
||||
{
|
||||
my $cChar = substr($strString, $iIndex, 1);
|
||||
|
||||
# These characters are reproduced verbatim
|
||||
if (($cChar ge 'A' && $cChar le 'Z') || ($cChar ge 'a' && $cChar le 'z') || ($cChar ge '0' && $cChar le '9') ||
|
||||
$cChar eq '_' || $cChar eq '-' || $cChar eq '~' || $cChar eq '.' || ($bPath && $cChar eq '/'))
|
||||
{
|
||||
$strEncodedString .= $cChar;
|
||||
}
|
||||
# Forward slash is encoded
|
||||
elsif ($cChar eq '/')
|
||||
{
|
||||
$strEncodedString .= '%2F';
|
||||
}
|
||||
# All other characters are hex-encoded
|
||||
else
|
||||
{
|
||||
$strEncodedString .= sprintf('%%%02X', ord($cChar));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $strEncodedString;
|
||||
}
|
||||
|
||||
push @EXPORT, qw(httpUriEncode);
|
||||
|
||||
1;
|
||||
@ -1,163 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# XML Helper Functions
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Common::Xml;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use XML::LibXML;
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlParse - parse a string into an xml document and return the root node
|
||||
####################################################################################################################################
|
||||
use constant XML_HEADER => '<?xml version="1.0" encoding="UTF-8"?>';
|
||||
push @EXPORT, qw(XML_HEADER);
|
||||
|
||||
####################################################################################################################################
|
||||
# Convert a string to xml so that it is suitable to be appended into xml
|
||||
####################################################################################################################################
|
||||
sub xmlFromText
|
||||
{
|
||||
my $strText = shift;
|
||||
|
||||
return XML::LibXML::Text->new($strText)->toString();
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlFromText);
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlParse - parse a string into an xml document and return the root node
|
||||
####################################################################################################################################
|
||||
sub xmlParse
|
||||
{
|
||||
my $rstrXml = shift;
|
||||
|
||||
my $oXml = XML::LibXML->load_xml(string => $rstrXml)->documentElement();
|
||||
|
||||
return $oXml;
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlParse);
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlTagChildren - get all children that match the tag
|
||||
####################################################################################################################################
|
||||
sub xmlTagChildren
|
||||
{
|
||||
my $oXml = shift;
|
||||
my $strTag = shift;
|
||||
|
||||
return $oXml->getChildrenByTagName($strTag);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlTagChildren);
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlTagText - get the text content for a tag, error if the tag is required and does not exist
|
||||
####################################################################################################################################
|
||||
sub xmlTagText
|
||||
{
|
||||
my $oXml = shift;
|
||||
my $strTag = shift;
|
||||
my $bRequired = shift;
|
||||
# my $strDefault = shift;
|
||||
|
||||
# Get the tag or tags
|
||||
my @oyTag = $oXml->getElementsByTagName($strTag);
|
||||
|
||||
# Error if the tag does not exist and is required
|
||||
if (@oyTag > 1)
|
||||
{
|
||||
confess &log(ERROR, @oyTag . " '${strTag}' tag(s) exist, but only one was expected", ERROR_FORMAT);
|
||||
}
|
||||
elsif (@oyTag == 0)
|
||||
{
|
||||
if (!defined($bRequired) || $bRequired)
|
||||
{
|
||||
confess &log(ERROR, "tag '${strTag}' does not exist", ERROR_FORMAT);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return $oyTag[0]->textContent();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlTagText);
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlTagBool - get the boolean content for a tag, error if the tag is required and does not exist or is not boolean
|
||||
####################################################################################################################################
|
||||
sub xmlTagBool
|
||||
{
|
||||
my $oXml = shift;
|
||||
my $strTag = shift;
|
||||
my $bRequired = shift;
|
||||
# my $strDefault = shift;
|
||||
|
||||
# Test content for boolean value
|
||||
my $strContent = xmlTagText($oXml, $strTag, $bRequired);
|
||||
|
||||
if (defined($strContent))
|
||||
{
|
||||
if ($strContent eq 'true')
|
||||
{
|
||||
return true;
|
||||
}
|
||||
elsif ($strContent eq 'false')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
confess &log(ERROR, "invalid boolean value '${strContent}' for tag '${strTag}'", ERROR_FORMAT);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlTagBool);
|
||||
|
||||
####################################################################################################################################
|
||||
# xmlTagInt - get the integer content for a tag, error if the tag is required and does not exist or is not an integer
|
||||
####################################################################################################################################
|
||||
sub xmlTagInt
|
||||
{
|
||||
my $oXml = shift;
|
||||
my $strTag = shift;
|
||||
my $bRequired = shift;
|
||||
# my $strDefault = shift;
|
||||
|
||||
# Test content for boolean value
|
||||
my $iContent = xmlTagText($oXml, $strTag, $bRequired);
|
||||
|
||||
if (defined($iContent))
|
||||
{
|
||||
eval
|
||||
{
|
||||
$iContent = $iContent + 0;
|
||||
return 1;
|
||||
}
|
||||
or do
|
||||
{
|
||||
confess &log(ERROR, "invalid integer value '${iContent}' for tag '${strTag}'", ERROR_FORMAT);
|
||||
}
|
||||
}
|
||||
|
||||
return $iContent;
|
||||
}
|
||||
|
||||
push @EXPORT, qw(xmlTagInt);
|
||||
|
||||
1;
|
||||
@ -74,8 +74,6 @@ sub libcAutoExportTag
|
||||
checksum =>
|
||||
[
|
||||
'pageChecksum',
|
||||
'pageChecksumBufferTest',
|
||||
'pageChecksumTest',
|
||||
],
|
||||
|
||||
config =>
|
||||
@ -352,7 +350,7 @@ sub libcAutoExportTag
|
||||
|
||||
storage =>
|
||||
[
|
||||
'storagePosixPathRemove',
|
||||
'storageRepoFree',
|
||||
],
|
||||
|
||||
test =>
|
||||
|
||||
@ -11,7 +11,6 @@ use Carp qw(confess);
|
||||
use pgBackRest::Backup::File;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Storage::Local;
|
||||
use pgBackRest::Protocol::Base::Master;
|
||||
use pgBackRest::Protocol::Base::Minion;
|
||||
use pgBackRest::Protocol::Command::Minion;
|
||||
|
||||
@ -91,11 +91,12 @@ sub init
|
||||
my $oSourceFileIo = $oStorage->openRead(@{shift()});
|
||||
|
||||
# If the source file exists
|
||||
if (defined($oSourceFileIo))
|
||||
if (defined($oSourceFileIo) && (!defined($oSourceFileIo->{oStorageCRead}) || $oSourceFileIo->open()))
|
||||
{
|
||||
$self->outputWrite(true);
|
||||
|
||||
$oStorage->copy($oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo));
|
||||
$oStorage->copy(
|
||||
$oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo), {bSourceOpen => true});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -13,11 +13,10 @@ use File::Basename qw(basename);
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::LibC qw(:storage);
|
||||
use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::Remote;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Helper;
|
||||
use pgBackRest::Storage::Local;
|
||||
|
||||
####################################################################################################################################
|
||||
# Storage constants
|
||||
@ -60,9 +59,8 @@ sub storageDb
|
||||
{
|
||||
if (isDbLocal({iRemoteIdx => $iRemoteIdx}))
|
||||
{
|
||||
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Local(
|
||||
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)), new pgBackRest::Storage::Posix::Driver(),
|
||||
{strTempExtension => STORAGE_TEMP_EXT, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
|
||||
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Storage(
|
||||
STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -81,54 +79,6 @@ sub storageDb
|
||||
|
||||
push @EXPORT, qw(storageDb);
|
||||
|
||||
####################################################################################################################################
|
||||
# storageRepoRule - rules for paths in the repository
|
||||
####################################################################################################################################
|
||||
sub storageRepoRule
|
||||
{
|
||||
my $strRule = shift;
|
||||
my $strFile = shift;
|
||||
my $strStanza = shift;
|
||||
|
||||
# Result path and file
|
||||
my $strResultFile;
|
||||
|
||||
# Return archive path
|
||||
if ($strRule eq STORAGE_REPO_ARCHIVE)
|
||||
{
|
||||
$strResultFile = "archive" . (defined($strStanza) ? "/${strStanza}" : '');
|
||||
|
||||
# If file is not defined nothing further to do
|
||||
if (defined($strFile))
|
||||
{
|
||||
my ($strArchiveId, $strWalFile) = split('/', $strFile);
|
||||
|
||||
# If a WAL file (including .backup)
|
||||
if (defined($strWalFile) && $strWalFile =~ /^[0-F]{24}/)
|
||||
{
|
||||
$strResultFile .= "/${strArchiveId}/" . substr($strWalFile, 0, 16) . "/${strWalFile}";
|
||||
}
|
||||
# Else other files
|
||||
else
|
||||
{
|
||||
$strResultFile .= "/${strFile}";
|
||||
}
|
||||
}
|
||||
}
|
||||
# Return backup path
|
||||
elsif ($strRule eq STORAGE_REPO_BACKUP)
|
||||
{
|
||||
$strResultFile = "backup" . (defined($strStanza) ? "/${strStanza}" : '') . (defined($strFile) ? "/${strFile}" : '');
|
||||
}
|
||||
# Else error
|
||||
else
|
||||
{
|
||||
confess &log(ASSERT, "invalid " . STORAGE_REPO . " storage rule ${strRule}");
|
||||
}
|
||||
|
||||
return $strResultFile;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# storageRepo - get repository storage
|
||||
####################################################################################################################################
|
||||
@ -146,85 +96,18 @@ sub storageRepo
|
||||
{name => 'strStanza', optional => true, trace => true},
|
||||
);
|
||||
|
||||
if (!defined($strStanza))
|
||||
{
|
||||
if (cfgOptionValid(CFGOPT_STANZA) && cfgOptionTest(CFGOPT_STANZA))
|
||||
{
|
||||
$strStanza = cfgOption(CFGOPT_STANZA);
|
||||
}
|
||||
else
|
||||
{
|
||||
$strStanza = STORAGE_REPO;
|
||||
}
|
||||
}
|
||||
|
||||
# Create storage if not defined
|
||||
if (!defined($hStorage->{&STORAGE_REPO}{$strStanza}))
|
||||
if (!defined($hStorage->{&STORAGE_REPO}))
|
||||
{
|
||||
if (isRepoLocal())
|
||||
{
|
||||
# Path rules
|
||||
my $hRule =
|
||||
{
|
||||
&STORAGE_REPO_ARCHIVE =>
|
||||
{
|
||||
fnRule => \&storageRepoRule,
|
||||
xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,
|
||||
},
|
||||
&STORAGE_REPO_BACKUP =>
|
||||
{
|
||||
fnRule => \&storageRepoRule,
|
||||
xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,
|
||||
},
|
||||
};
|
||||
|
||||
# Create the driver
|
||||
my $oDriver;
|
||||
|
||||
if (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3))
|
||||
{
|
||||
require pgBackRest::Storage::S3::Driver;
|
||||
|
||||
$oDriver = new pgBackRest::Storage::S3::Driver(
|
||||
cfgOption(CFGOPT_REPO_S3_BUCKET), cfgOption(CFGOPT_REPO_S3_ENDPOINT), cfgOption(CFGOPT_REPO_S3_REGION),
|
||||
cfgOption(CFGOPT_REPO_S3_KEY), cfgOption(CFGOPT_REPO_S3_KEY_SECRET),
|
||||
{strHost => cfgOption(CFGOPT_REPO_S3_HOST, false), bVerifySsl => cfgOption(CFGOPT_REPO_S3_VERIFY_TLS, false),
|
||||
strCaPath => cfgOption(CFGOPT_REPO_S3_CA_PATH, false),
|
||||
strCaFile => cfgOption(CFGOPT_REPO_S3_CA_FILE, false), lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),
|
||||
strSecurityToken => cfgOption(CFGOPT_REPO_S3_TOKEN, false)});
|
||||
}
|
||||
elsif (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_CIFS))
|
||||
{
|
||||
require pgBackRest::Storage::Cifs::Driver;
|
||||
|
||||
$oDriver = new pgBackRest::Storage::Cifs::Driver();
|
||||
}
|
||||
else
|
||||
{
|
||||
$oDriver = new pgBackRest::Storage::Posix::Driver();
|
||||
}
|
||||
|
||||
# Set the encryption for the repo
|
||||
my $strCipherType;
|
||||
my $strCipherPass;
|
||||
|
||||
# If the encryption is not the default (none) then set the user-defined passphrase and magic based on the type
|
||||
if (cfgOption(CFGOPT_REPO_CIPHER_TYPE) ne CFGOPTVAL_REPO_CIPHER_TYPE_NONE)
|
||||
{
|
||||
$strCipherType = cfgOption(CFGOPT_REPO_CIPHER_TYPE);
|
||||
$strCipherPass = cfgOption(CFGOPT_REPO_CIPHER_PASS);
|
||||
}
|
||||
|
||||
# Create local storage
|
||||
$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Storage::Local(
|
||||
cfgOption(CFGOPT_REPO_PATH), $oDriver,
|
||||
{strTempExtension => STORAGE_TEMP_EXT, hRule => $hRule, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),
|
||||
strCipherType => $strCipherType, strCipherPassUser => $strCipherPass});
|
||||
$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(
|
||||
STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
|
||||
}
|
||||
else
|
||||
{
|
||||
# Create remote storage
|
||||
$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Protocol::Storage::Remote(
|
||||
$hStorage->{&STORAGE_REPO} = new pgBackRest::Protocol::Storage::Remote(
|
||||
protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP));
|
||||
}
|
||||
}
|
||||
@ -233,7 +116,7 @@ sub storageRepo
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}{$strStanza}, trace => true},
|
||||
{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
@ -249,6 +132,8 @@ sub storageRepoCacheClear
|
||||
|
||||
delete($hStorage->{&STORAGE_REPO});
|
||||
|
||||
storageRepoFree();
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
@ -15,7 +15,6 @@ use pgBackRest::Config::Config;
|
||||
use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::File;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
@ -195,29 +194,10 @@ sub openRead
|
||||
{name => 'rhParam', required => false},
|
||||
);
|
||||
|
||||
# Determine whether protocol compress will be used
|
||||
my $bProtocolCompress = protocolCompress($rhParam);
|
||||
|
||||
# Compress on the remote side
|
||||
if ($bProtocolCompress)
|
||||
{
|
||||
push(
|
||||
@{$rhParam->{rhyFilter}},
|
||||
{strClass => STORAGE_FILTER_GZIP,
|
||||
rxyParam => [{iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false}]});
|
||||
}
|
||||
|
||||
my $oSourceFileIo =
|
||||
$self->{oProtocol}->cmdExecute(OP_STORAGE_OPEN_READ, [$strFileExp, $rhParam]) ?
|
||||
new pgBackRest::Protocol::Storage::File($self->{oProtocol}) : undef;
|
||||
|
||||
# Decompress on the local side
|
||||
if ($bProtocolCompress)
|
||||
{
|
||||
$oSourceFileIo = new pgBackRest::Storage::Filter::Gzip(
|
||||
$oSourceFileIo, {strCompressType => STORAGE_DECOMPRESS, bWantGzip => false});
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
@ -247,28 +227,10 @@ sub openWrite
|
||||
{name => 'rhParam', required => false},
|
||||
);
|
||||
|
||||
# Determine whether protocol compress will be used
|
||||
my $bProtocolCompress = protocolCompress($rhParam);
|
||||
|
||||
# Decompress on the remote side
|
||||
if ($bProtocolCompress)
|
||||
{
|
||||
push(
|
||||
@{$rhParam->{rhyFilter}},
|
||||
{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS, bWantGzip => false}]});
|
||||
}
|
||||
|
||||
# Open the remote file
|
||||
$self->{oProtocol}->cmdWrite(OP_STORAGE_OPEN_WRITE, [$strFileExp, $rhParam]);
|
||||
my $oDestinationFileIo = new pgBackRest::Protocol::Storage::File($self->{oProtocol});
|
||||
|
||||
# Compress on local side
|
||||
if ($bProtocolCompress)
|
||||
{
|
||||
$oDestinationFileIo = new pgBackRest::Storage::Filter::Gzip(
|
||||
$oDestinationFileIo, {iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false});
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
@ -367,24 +329,6 @@ sub cipherPassUser
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Used internally to determine if protocol compression should be enabled
|
||||
####################################################################################################################################
|
||||
sub protocolCompress
|
||||
{
|
||||
my $rhParam = shift;
|
||||
|
||||
my $bProtocolCompress = false;
|
||||
|
||||
if (defined($rhParam->{bProtocolCompress}))
|
||||
{
|
||||
$bProtocolCompress = $rhParam->{bProtocolCompress} && cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK) > 0 ? true : false;
|
||||
delete($rhParam->{bProtocolCompress});
|
||||
}
|
||||
|
||||
return $bProtocolCompress;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# getters
|
||||
####################################################################################################################################
|
||||
|
||||
@ -227,8 +227,7 @@ sub manifestLoad
|
||||
|
||||
# Copy the backup manifest to the db cluster path
|
||||
storageDb()->copy(
|
||||
storageRepo()->openRead(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST, {bProtocolCompress => true,
|
||||
strCipherPass => $strCipherPass}),
|
||||
storageRepo()->openRead(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST, {strCipherPass => $strCipherPass}),
|
||||
$self->{strDbClusterPath} . '/' . FILE_MANIFEST);
|
||||
|
||||
# Load the manifest into a hash
|
||||
@ -1089,7 +1088,7 @@ sub process
|
||||
|
||||
# Copy backup info, load it, then delete
|
||||
$oStorageDb->copy(
|
||||
storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO, {bProtocolCompress => true}),
|
||||
storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO),
|
||||
$self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO);
|
||||
|
||||
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strDbClusterPath}, false, undef, {oStorage => storageDb()});
|
||||
|
||||
@ -20,9 +20,6 @@ use pgBackRest::Common::String;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
####################################################################################################################################
|
||||
|
||||
@ -294,7 +294,7 @@ sub stanzaDelete
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->stanzaDelete');
|
||||
|
||||
my $strStanza = cfgOption(CFGOPT_STANZA);
|
||||
my $oStorageRepo = storageRepo({strStanza => $strStanza});
|
||||
my $oStorageRepo = storageRepo();
|
||||
|
||||
# If at least an archive or backup directory exists for the stanza, then continue, else nothing to do
|
||||
if ($oStorageRepo->pathExists(STORAGE_REPO_ARCHIVE) || $oStorageRepo->pathExists(STORAGE_REPO_BACKUP))
|
||||
@ -342,8 +342,8 @@ sub stanzaDelete
|
||||
}
|
||||
|
||||
# Recursively remove the stanza archive and backup directories
|
||||
$oStorageRepo->remove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true});
|
||||
$oStorageRepo->remove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true});
|
||||
$oStorageRepo->pathRemove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true});
|
||||
$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true});
|
||||
|
||||
# Remove the stop file so processes can run.
|
||||
lockStart();
|
||||
|
||||
@ -16,6 +16,17 @@ use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Base;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# Storage constants
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_LOCAL => '<LOCAL>';
|
||||
push @EXPORT, qw(STORAGE_LOCAL);
|
||||
|
||||
use constant STORAGE_S3 => 's3';
|
||||
push @EXPORT, qw(STORAGE_S3);
|
||||
use constant STORAGE_POSIX => 'posix';
|
||||
push @EXPORT, qw(STORAGE_POSIX);
|
||||
|
||||
####################################################################################################################################
|
||||
# Compress constants
|
||||
####################################################################################################################################
|
||||
@ -34,6 +45,16 @@ use constant STORAGE_DECRYPT => 'decrypt'
|
||||
use constant CIPHER_MAGIC => 'Salted__';
|
||||
push @EXPORT, qw(CIPHER_MAGIC);
|
||||
|
||||
####################################################################################################################################
|
||||
# Filter constants
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_FILTER_CIPHER_BLOCK => 'pgBackRest::Storage::Filter::CipherBlock';
|
||||
push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);
|
||||
use constant STORAGE_FILTER_GZIP => 'pgBackRest::Storage::Filter::Gzip';
|
||||
push @EXPORT, qw(STORAGE_FILTER_GZIP);
|
||||
use constant STORAGE_FILTER_SHA => 'pgBackRest::Storage::Filter::Sha';
|
||||
push @EXPORT, qw(STORAGE_FILTER_SHA);
|
||||
|
||||
####################################################################################################################################
|
||||
# Capability constants
|
||||
####################################################################################################################################
|
||||
@ -78,9 +99,10 @@ sub new
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
####################################################################################################################################
|
||||
# copy - copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed
|
||||
# instead of file names.
|
||||
# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of
|
||||
# file names.
|
||||
####################################################################################################################################
|
||||
sub copy
|
||||
{
|
||||
@ -92,47 +114,62 @@ sub copy
|
||||
$strOperation,
|
||||
$xSourceFile,
|
||||
$xDestinationFile,
|
||||
$bSourceOpen,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->copy', \@_,
|
||||
{name => 'xSourceFile', required => false},
|
||||
{name => 'xDestinationFile', required => false},
|
||||
{name => 'xDestinationFile'},
|
||||
{name => 'bSourceOpen', optional => true, default => false},
|
||||
);
|
||||
|
||||
# Was the file copied?
|
||||
# Is source/destination an IO object or a file expression?
|
||||
my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;
|
||||
|
||||
# Does the source file exist?
|
||||
my $bResult = false;
|
||||
|
||||
# Is source an IO object or a file expression?
|
||||
my $oSourceFileIo =
|
||||
defined($xSourceFile) ?
|
||||
(ref($xSourceFile) ? $xSourceFile : $self->openRead($self->pathGet($xSourceFile))) : undef;
|
||||
|
||||
# Proceed if source file exists
|
||||
# Copy if the source file exists
|
||||
if (defined($oSourceFileIo))
|
||||
{
|
||||
# Is destination an IO object or a file expression?
|
||||
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($self->pathGet($xDestinationFile));
|
||||
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);
|
||||
|
||||
# Copy the data
|
||||
my $lSizeRead;
|
||||
|
||||
do
|
||||
# Use C copy if source and destination are C objects
|
||||
if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))
|
||||
{
|
||||
# Read data
|
||||
my $tBuffer = '';
|
||||
|
||||
$lSizeRead = $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
|
||||
$oDestinationFileIo->write(\$tBuffer);
|
||||
$bResult = $self->{oStorageC}->copy(
|
||||
$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;
|
||||
}
|
||||
while ($lSizeRead != 0);
|
||||
else
|
||||
{
|
||||
# Open the source file if it is a C object
|
||||
$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;
|
||||
|
||||
# Close files
|
||||
$oSourceFileIo->close();
|
||||
$oDestinationFileIo->close();
|
||||
if ($bResult)
|
||||
{
|
||||
# Open the destination file if it is a C object
|
||||
if (defined($oDestinationFileIo->{oStorageCWrite}))
|
||||
{
|
||||
$oDestinationFileIo->open();
|
||||
}
|
||||
|
||||
# File was copied
|
||||
$bResult = true;
|
||||
# Copy the data
|
||||
do
|
||||
{
|
||||
# Read data
|
||||
my $tBuffer = '';
|
||||
|
||||
$oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
|
||||
$oDestinationFileIo->write(\$tBuffer);
|
||||
}
|
||||
while (!$oSourceFileIo->eof());
|
||||
|
||||
# Close files
|
||||
$oSourceFileIo->close();
|
||||
$oDestinationFileIo->close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return logDebugReturn
|
||||
|
||||
@ -1,55 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# CIFS Storage
|
||||
#
|
||||
# Implements storage functions for Posix-compliant file systems.
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Cifs::Driver;
|
||||
use parent 'pgBackRest::Storage::Posix::Driver';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_CIFS_DRIVER => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_CIFS_DRIVER);
|
||||
|
||||
####################################################################################################################################
|
||||
# pathSync - CIFS does not support path sync so this is a noop
|
||||
####################################################################################################################################
|
||||
sub pathSync
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathSync', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters/Setters
|
||||
####################################################################################################################################
|
||||
sub capability {shift eq STORAGE_CAPABILITY_SIZE_DIFF ? true : false}
|
||||
sub className {STORAGE_CIFS_DRIVER}
|
||||
|
||||
1;
|
||||
@ -1,177 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Block Cipher Filter
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Filter::CipherBlock;
|
||||
use parent 'pgBackRest::Common::Io::Filter';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Base;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_FILTER_CIPHER_BLOCK => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oParent,
|
||||
$strCipherType,
|
||||
$tCipherPass,
|
||||
$strMode,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oParent', trace => true},
|
||||
{name => 'strCipherType', trace => true},
|
||||
{name => 'tCipherPass', trace => true},
|
||||
{name => 'strMode', optional => true, default => STORAGE_ENCRYPT, trace => true},
|
||||
);
|
||||
|
||||
# Bless with new class
|
||||
my $self = $class->SUPER::new($oParent);
|
||||
bless $self, $class;
|
||||
|
||||
# Check mode is valid
|
||||
$self->{strMode} = $strMode;
|
||||
|
||||
if (!($self->{strMode} eq STORAGE_ENCRYPT || $self->{strMode} eq STORAGE_DECRYPT))
|
||||
{
|
||||
confess &log(ASSERT, "unknown cipher mode: $self->{strMode}");
|
||||
}
|
||||
|
||||
# Set read/write
|
||||
$self->{bWrite} = false;
|
||||
|
||||
# Create cipher object
|
||||
$self->{oCipher} = new pgBackRest::LibC::Cipher::Block(
|
||||
$self->{strMode} eq STORAGE_ENCRYPT ? CIPHER_MODE_ENCRYPT : CIPHER_MODE_DECRYPT, $strCipherType, $tCipherPass,
|
||||
length($tCipherPass));
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# read - encrypt/decrypt data
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
my $iSize = shift;
|
||||
|
||||
# Return 0 if all data has been read
|
||||
return 0 if $self->eof();
|
||||
|
||||
# Loop until required bytes have been read
|
||||
my $tBufferRead = '';
|
||||
my $iBufferReadSize = 0;
|
||||
|
||||
do
|
||||
{
|
||||
# Read data
|
||||
my $tCipherBuffer;
|
||||
my $iActualSize = $self->SUPER::read(\$tCipherBuffer, $iSize);
|
||||
|
||||
# If something was read, then process it
|
||||
if ($iActualSize > 0)
|
||||
{
|
||||
$tBufferRead .= $self->{oCipher}->process($tCipherBuffer);
|
||||
}
|
||||
|
||||
# If eof then flush the remaining data
|
||||
if ($self->eof())
|
||||
{
|
||||
$tBufferRead .= $self->{oCipher}->flush();
|
||||
}
|
||||
|
||||
# Get the current size of the read buffer
|
||||
$iBufferReadSize = length($tBufferRead);
|
||||
}
|
||||
while ($iBufferReadSize < $iSize && !$self->eof());
|
||||
|
||||
# Append to the read buffer
|
||||
$$rtBuffer .= $tBufferRead;
|
||||
|
||||
# Return the actual size read
|
||||
return $iBufferReadSize;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# write - encrypt/decrypt data
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
|
||||
# Set write flag so close will flush buffer
|
||||
$self->{bWrite} = true;
|
||||
|
||||
# Write the buffer if defined
|
||||
my $tCipherBuffer;
|
||||
|
||||
if (defined($$rtBuffer))
|
||||
{
|
||||
$tCipherBuffer = $self->{oCipher}->process($$rtBuffer);
|
||||
}
|
||||
|
||||
# Call the io method. If $rtBuffer is undefined, then this is expected to error.
|
||||
$self->SUPER::write(\$tCipherBuffer);
|
||||
|
||||
return length($$rtBuffer);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Only close the object if not already closed
|
||||
if ($self->{oCipher})
|
||||
{
|
||||
# Flush the write buffer
|
||||
if ($self->{bWrite})
|
||||
{
|
||||
my $tCipherBuffer = $self->{oCipher}->flush();
|
||||
$self->SUPER::write(\$tCipherBuffer);
|
||||
}
|
||||
|
||||
undef($self->{oCipher});
|
||||
|
||||
# Close io
|
||||
return $self->SUPER::close();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,264 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# GZIP Filter
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Filter::Gzip;
|
||||
use parent 'pgBackRest::Common::Io::Filter';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Compress::Raw::Zlib qw(WANT_GZIP MAX_WBITS Z_OK Z_BUF_ERROR Z_DATA_ERROR Z_STREAM_END);
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Base;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_FILTER_GZIP => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_FILTER_GZIP);
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oParent,
|
||||
$bWantGzip,
|
||||
$strCompressType,
|
||||
$iLevel,
|
||||
$lCompressBufferMax,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oParent', trace => true},
|
||||
{name => 'bWantGzip', optional => true, default => true, trace => true},
|
||||
{name => 'strCompressType', optional => true, default => STORAGE_COMPRESS, trace => true},
|
||||
{name => 'iLevel', optional => true, default => 6, trace => true},
|
||||
{name => 'lCompressBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
|
||||
);
|
||||
|
||||
# Bless with new class
|
||||
my $self = $class->SUPER::new($oParent);
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{bWantGzip} = $bWantGzip;
|
||||
$self->{iLevel} = $iLevel;
|
||||
$self->{lCompressBufferMax} = $lCompressBufferMax;
|
||||
$self->{strCompressType} = $strCompressType;
|
||||
|
||||
# Set read/write
|
||||
$self->{bWrite} = false;
|
||||
|
||||
# Create the zlib object
|
||||
my $iZLibStatus;
|
||||
|
||||
if ($self->{strCompressType} eq STORAGE_COMPRESS)
|
||||
{
|
||||
($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Deflate(
|
||||
WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Level => $self->{iLevel},
|
||||
Bufsize => $self->{lCompressBufferMax}, AppendOutput => 1);
|
||||
|
||||
$self->{tCompressedBuffer} = undef;
|
||||
}
|
||||
else
|
||||
{
|
||||
($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Inflate(
|
||||
WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Bufsize => $self->{lCompressBufferMax},
|
||||
LimitOutput => 1, AppendOutput => 1);
|
||||
|
||||
$self->{tUncompressedBuffer} = undef;
|
||||
$self->{lUncompressedBufferSize} = 0;
|
||||
}
|
||||
|
||||
$self->errorCheck($iZLibStatus);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# errorCheck - check status code for errors
|
||||
####################################################################################################################################
|
||||
sub errorCheck
|
||||
{
|
||||
my $self = shift;
|
||||
my $iZLibStatus = shift;
|
||||
|
||||
if (!($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR))
|
||||
{
|
||||
logErrorResult(
|
||||
$self->{bWrite} ? ERROR_FILE_WRITE : ERROR_FILE_READ,
|
||||
'unable to ' . ($self->{strCompressType} eq STORAGE_COMPRESS ? 'deflate' : 'inflate') . " '" .
|
||||
$self->parent()->name() . "'",
|
||||
$self->{oZLib}->msg());
|
||||
}
|
||||
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# read - compress/decompress data
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
my $iSize = shift;
|
||||
|
||||
if ($self->{strCompressType} eq STORAGE_COMPRESS)
|
||||
{
|
||||
return 0 if $self->eof();
|
||||
|
||||
my $lSizeBegin = defined($$rtBuffer) ? length($$rtBuffer) : 0;
|
||||
my $lUncompressedSize;
|
||||
my $lCompressedSize;
|
||||
|
||||
do
|
||||
{
|
||||
my $tUncompressedBuffer;
|
||||
$lUncompressedSize = $self->parent()->read(\$tUncompressedBuffer, $iSize);
|
||||
|
||||
if ($lUncompressedSize > 0)
|
||||
{
|
||||
$self->errorCheck($self->{oZLib}->deflate($tUncompressedBuffer, $$rtBuffer));
|
||||
}
|
||||
else
|
||||
{
|
||||
$self->errorCheck($self->{oZLib}->flush($$rtBuffer));
|
||||
}
|
||||
|
||||
$lCompressedSize = length($$rtBuffer) - $lSizeBegin;
|
||||
}
|
||||
while ($lUncompressedSize > 0 && $lCompressedSize < $iSize);
|
||||
|
||||
# Return the actual size read
|
||||
return $lCompressedSize;
|
||||
}
|
||||
else
|
||||
{
|
||||
# If the local buffer size is not large enough to satisfy the request and there is still data to decompress
|
||||
while ($self->{lUncompressedBufferSize} < $iSize && !$self->parent()->eof())
|
||||
{
|
||||
if (!defined($self->{tCompressedBuffer}) || length($self->{tCompressedBuffer}) == 0)
|
||||
{
|
||||
$self->parent()->read(\$self->{tCompressedBuffer}, $self->{lCompressBufferMax});
|
||||
}
|
||||
|
||||
my $iZLibStatus = $self->{oZLib}->inflate($self->{tCompressedBuffer}, $self->{tUncompressedBuffer});
|
||||
$self->{lUncompressedBufferSize} = length($self->{tUncompressedBuffer});
|
||||
|
||||
last if $iZLibStatus == Z_STREAM_END;
|
||||
|
||||
$self->errorCheck($iZLibStatus);
|
||||
}
|
||||
|
||||
# Actual size is the lesser of the local buffer size or requested size - if the local buffer is smaller than the requested
|
||||
# size it means that there was nothing more to be read
|
||||
my $iActualSize = $self->{lUncompressedBufferSize} < $iSize ? $self->{lUncompressedBufferSize} : $iSize;
|
||||
|
||||
# Append to the request buffer
|
||||
$$rtBuffer .= substr($self->{tUncompressedBuffer}, 0, $iActualSize);
|
||||
|
||||
# Truncate local buffer
|
||||
$self->{tUncompressedBuffer} = substr($self->{tUncompressedBuffer}, $iActualSize);
|
||||
$self->{lUncompressedBufferSize} -= $iActualSize;
|
||||
|
||||
# Return the actual size read
|
||||
return $iActualSize;
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# write - compress/decompress data
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
|
||||
$self->{bWrite} = true;
|
||||
|
||||
if ($self->{strCompressType} eq STORAGE_COMPRESS)
|
||||
{
|
||||
# Compress the data
|
||||
$self->errorCheck($self->{oZLib}->deflate($$rtBuffer, $self->{tCompressedBuffer}));
|
||||
|
||||
# Only write when buffer is full
|
||||
if (length($self->{tCompressedBuffer}) > $self->{lCompressBufferMax})
|
||||
{
|
||||
$self->parent()->write(\$self->{tCompressedBuffer});
|
||||
$self->{tCompressedBuffer} = undef;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
my $tCompressedBuffer = $$rtBuffer;
|
||||
|
||||
while (length($tCompressedBuffer) > 0)
|
||||
{
|
||||
my $tUncompressedBuffer;
|
||||
|
||||
my $iZLibStatus = $self->{oZLib}->inflate($tCompressedBuffer, $tUncompressedBuffer);
|
||||
$self->parent()->write(\$tUncompressedBuffer);
|
||||
|
||||
last if $iZLibStatus == Z_STREAM_END;
|
||||
|
||||
$self->errorCheck($iZLibStatus);
|
||||
}
|
||||
}
|
||||
|
||||
# Return bytes written
|
||||
return length($$rtBuffer);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->{oZLib}))
|
||||
{
|
||||
# Flush the write buffer
|
||||
if ($self->{bWrite})
|
||||
{
|
||||
if ($self->{strCompressType} eq STORAGE_COMPRESS)
|
||||
{
|
||||
# Flush out last compressed bytes
|
||||
$self->errorCheck($self->{oZLib}->flush($self->{tCompressedBuffer}));
|
||||
|
||||
# Write last compressed bytes
|
||||
$self->parent()->write(\$self->{tCompressedBuffer});
|
||||
}
|
||||
}
|
||||
|
||||
undef($self->{oZLib});
|
||||
|
||||
# Close io
|
||||
return $self->parent()->close();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,124 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# SHA Filter
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Filter::Sha;
|
||||
use parent 'pgBackRest::Common::Io::Filter';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_FILTER_SHA => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_FILTER_SHA);
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oParent,
|
||||
$strAlgorithm,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oParent', trace => true},
|
||||
{name => 'strAlgorithm', optional => true, default => 'sha1', trace => true},
|
||||
);
|
||||
|
||||
# Bless with new class
|
||||
my $self = $class->SUPER::new($oParent);
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{strAlgorithm} = $strAlgorithm;
|
||||
|
||||
# Create SHA object
|
||||
$self->{oSha} = new pgBackRest::LibC::Crypto::Hash($self->{strAlgorithm});
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# read - calculate sha digest
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
my $iSize = shift;
|
||||
|
||||
# Call the io method
|
||||
my $tShaBuffer;
|
||||
my $iActualSize = $self->parent()->read(\$tShaBuffer, $iSize);
|
||||
|
||||
# Calculate sha for the returned buffer
|
||||
if ($iActualSize > 0)
|
||||
{
|
||||
$self->{oSha}->process($tShaBuffer);
|
||||
$$rtBuffer .= $tShaBuffer;
|
||||
}
|
||||
|
||||
# Return the actual size read
|
||||
return $iActualSize;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# write - calculate sha digest
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
|
||||
# Calculate sha for the buffer
|
||||
$self->{oSha}->process($$rtBuffer);
|
||||
|
||||
# Call the io method
|
||||
return $self->parent()->write($rtBuffer);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->{oSha}))
|
||||
{
|
||||
# Set result
|
||||
$self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->result());
|
||||
|
||||
# Delete the sha object
|
||||
delete($self->{oSha});
|
||||
|
||||
# Close io
|
||||
return $self->parent->close();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
1;
|
||||
@ -13,16 +13,10 @@ use File::Basename qw(basename);
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
use pgBackRest::Storage::Local;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Storage;
|
||||
use pgBackRest::Version;
|
||||
|
||||
####################################################################################################################################
|
||||
# Storage constants
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_LOCAL => '<LOCAL>';
|
||||
push @EXPORT, qw(STORAGE_LOCAL);
|
||||
|
||||
####################################################################################################################################
|
||||
# Compression extension
|
||||
####################################################################################################################################
|
||||
@ -35,11 +29,6 @@ use constant COMPRESS_EXT => 'gz';
|
||||
use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp';
|
||||
push @EXPORT, qw(STORAGE_TEMP_EXT);
|
||||
|
||||
####################################################################################################################################
|
||||
# Cache storage so it can be retrieved quickly
|
||||
####################################################################################################################################
|
||||
my $hStorage;
|
||||
|
||||
####################################################################################################################################
|
||||
# storageLocal - get local storage
|
||||
#
|
||||
@ -49,32 +38,13 @@ my $hStorage;
|
||||
sub storageLocal
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::storageLocal', \@_,
|
||||
{name => 'strPath', default => '/', trace => true},
|
||||
);
|
||||
|
||||
# Create storage if not defined
|
||||
if (!defined($hStorage->{&STORAGE_LOCAL}{$strPath}))
|
||||
{
|
||||
# Create local storage
|
||||
$hStorage->{&STORAGE_LOCAL}{$strPath} = new pgBackRest::Storage::Local(
|
||||
$strPath, new pgBackRest::Storage::Posix::Driver(),
|
||||
{strTempExtension => STORAGE_TEMP_EXT,
|
||||
lBufferMax => cfgOptionValid(CFGOPT_BUFFER_SIZE, false) ? cfgOption(CFGOPT_BUFFER_SIZE, false) : undef});
|
||||
}
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageLocal');
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oStorageLocal', value => $hStorage->{&STORAGE_LOCAL}{$strPath}, trace => true},
|
||||
{name => 'oStorageLocal', value => new pgBackRest::Storage::Storage(STORAGE_LOCAL), trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -1,963 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Posix Storage
|
||||
#
|
||||
# Implements storage functions for Posix-compliant file systems.
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Posix::Driver;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use File::Basename qw(basename dirname);
|
||||
use Fcntl qw(:mode);
|
||||
use File::stat qw{lstat};
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Posix::FileRead;
|
||||
use pgBackRest::Storage::Posix::FileWrite;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_POSIX_DRIVER => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_POSIX_DRIVER);
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Create the class hash
|
||||
my $self = {};
|
||||
bless $self, $class;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
(
|
||||
my $strOperation,
|
||||
$self->{bFileSync},
|
||||
$self->{bPathSync},
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'bFileSync', optional => true, default => true},
|
||||
{name => 'bPathSync', optional => true, default => true},
|
||||
);
|
||||
|
||||
# Set default temp extension
|
||||
$self->{strTempExtension} = 'tmp';
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# exists - check if a path or file exists
|
||||
####################################################################################################################################
|
||||
sub exists
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->exists', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
);
|
||||
|
||||
# Does the path/file exist?
|
||||
my $bExists = true;
|
||||
my $oStat = lstat($strFile);
|
||||
|
||||
# Use stat to test if file exists
|
||||
if (defined($oStat))
|
||||
{
|
||||
# Check that it is actually a file
|
||||
$bExists = !S_ISDIR($oStat->mode) ? true : false;
|
||||
}
|
||||
else
|
||||
{
|
||||
# If the error is not entry missing, then throw error
|
||||
if (!$OS_ERROR{ENOENT})
|
||||
{
|
||||
logErrorResult(ERROR_FILE_EXISTS, "unable to test if file '${strFile}' exists", $OS_ERROR);
|
||||
}
|
||||
|
||||
$bExists = false;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bExists', value => $bExists, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# info - get information for path/file
|
||||
####################################################################################################################################
|
||||
sub info
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPathFile,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->info', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Stat the path/file
|
||||
my $oInfo = lstat($strPathFile);
|
||||
|
||||
# Check for errors
|
||||
if (!defined($oInfo))
|
||||
{
|
||||
if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
|
||||
{
|
||||
logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to stat '${strPathFile}'", $OS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oInfo', value => $oInfo, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# linkCreate
|
||||
####################################################################################################################################
|
||||
sub linkCreate
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strSourcePathFile,
|
||||
$strDestinationLink,
|
||||
$bHard,
|
||||
$bPathCreate,
|
||||
$bIgnoreExists,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->linkCreate', \@_,
|
||||
{name => 'strSourcePathFile', trace => true},
|
||||
{name => 'strDestinationLink', trace => true},
|
||||
{name => 'bHard', optional=> true, default => false, trace => true},
|
||||
{name => 'bPathCreate', optional=> true, default => true, trace => true},
|
||||
{name => 'bIgnoreExists', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink)))
|
||||
{
|
||||
my $strMessage = "unable to create link '${strDestinationLink}'";
|
||||
|
||||
# If parent path or source is missing
|
||||
if ($OS_ERROR{ENOENT})
|
||||
{
|
||||
# Check if source is missing
|
||||
if (!$self->exists($strSourcePathFile))
|
||||
{
|
||||
confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING);
|
||||
}
|
||||
|
||||
if (!$bPathCreate)
|
||||
{
|
||||
confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING);
|
||||
}
|
||||
|
||||
# Create parent path
|
||||
$self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true});
|
||||
|
||||
# Create link
|
||||
$self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard});
|
||||
}
|
||||
# Else if link already exists
|
||||
elsif ($OS_ERROR{EEXIST})
|
||||
{
|
||||
if (!$bIgnoreExists)
|
||||
{
|
||||
confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# linkDestination - get destination of symlink
|
||||
####################################################################################################################################
|
||||
sub linkDestination
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strLink,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->linkDestination', \@_,
|
||||
{name => 'strLink', trace => true},
|
||||
);
|
||||
|
||||
# Get link destination
|
||||
my $strLinkDestination = readlink($strLink);
|
||||
|
||||
# Check for errors
|
||||
if (!defined($strLinkDestination))
|
||||
{
|
||||
logErrorResult(
|
||||
$OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to get destination for link ${strLink}", $OS_ERROR);
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strLinkDestination', value => $strLinkDestination, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# list - list all files/paths in path
|
||||
####################################################################################################################################
|
||||
sub list
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->list', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Working variables
|
||||
my @stryFileList;
|
||||
my $hPath;
|
||||
|
||||
# Attempt to open the path
|
||||
if (opendir($hPath, $strPath))
|
||||
{
|
||||
@stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath));
|
||||
close($hPath);
|
||||
}
|
||||
# Else process errors
|
||||
else
|
||||
{
|
||||
# Ignore the error if the file is missing and missing files should be ignored
|
||||
if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
|
||||
{
|
||||
logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to read path '${strPath}'", $OS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'stryFileList', value => \@stryFileList, ref => true, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# manifest - build path/file/link manifest starting with base path and including all subpaths
|
||||
####################################################################################################################################
|
||||
sub manifest
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$bIgnoreMissing,
|
||||
$strFilter,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->manifest', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
{name => 'strFilter', optional => true, trace => true},
|
||||
);
|
||||
|
||||
# Generate the manifest
|
||||
my $hManifest = {};
|
||||
$self->manifestRecurse($strPath, undef, 0, $hManifest, $bIgnoreMissing, $strFilter);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'hManifest', value => $hManifest, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
sub manifestRecurse
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$strSubPath,
|
||||
$iDepth,
|
||||
$hManifest,
|
||||
$bIgnoreMissing,
|
||||
$strFilter,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::manifestRecurse', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'strSubPath', required => false, trace => true},
|
||||
{name => 'iDepth', default => 0, trace => true},
|
||||
{name => 'hManifest', required => false, trace => true},
|
||||
{name => 'bIgnoreMissing', required => false, default => false, trace => true},
|
||||
{name => 'strFilter', required => false, trace => true},
|
||||
);
|
||||
|
||||
# Set operation and debug strings
|
||||
my $strPathRead = $strPath . (defined($strSubPath) ? "/${strSubPath}" : '');
|
||||
my $hPath;
|
||||
|
||||
# If this is the top level stat the path to discover if it is actually a file
|
||||
my $oPathInfo = $self->info($strPathRead, {bIgnoreMissing => $bIgnoreMissing});
|
||||
|
||||
if (defined($oPathInfo))
|
||||
{
|
||||
# If the initial path passed is a file then generate the manifest for just that file
|
||||
if ($iDepth == 0 && !S_ISDIR($oPathInfo->mode()))
|
||||
{
|
||||
$hManifest->{basename($strPathRead)} = $self->manifestStat($strPathRead);
|
||||
}
|
||||
# Else read as a normal directory
|
||||
else
|
||||
{
|
||||
# Get a list of all files in the path (including .)
|
||||
my @stryFileList = @{$self->list($strPathRead, {bIgnoreMissing => $iDepth != 0})};
|
||||
unshift(@stryFileList, '.');
|
||||
my $hFileStat = $self->manifestList($strPathRead, \@stryFileList, $strFilter);
|
||||
|
||||
# Loop through all subpaths/files in the path
|
||||
foreach my $strFile (keys(%{$hFileStat}))
|
||||
{
|
||||
my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : "/${strFile}"));
|
||||
$hManifest->{$strManifestFile} = $hFileStat->{$strFile};
|
||||
|
||||
# Recurse into directories
|
||||
if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.))
|
||||
{
|
||||
$self->manifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
sub manifestList
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$stryFile,
|
||||
$strFilter,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->manifestList', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'stryFile', trace => true},
|
||||
{name => 'strFilter', required => false, trace => true},
|
||||
);
|
||||
|
||||
my $hFileStat = {};
|
||||
|
||||
foreach my $strFile (@{$stryFile})
|
||||
{
|
||||
if ($strFile ne '.' && defined($strFilter) && $strFilter ne $strFile)
|
||||
{
|
||||
next;
|
||||
}
|
||||
|
||||
$hFileStat->{$strFile} = $self->manifestStat("${strPath}" . ($strFile eq qw(.) ? '' : "/${strFile}"));
|
||||
|
||||
if (!defined($hFileStat->{$strFile}))
|
||||
{
|
||||
delete($hFileStat->{$strFile});
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'hFileStat', value => $hFileStat, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
sub manifestStat
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->manifestStat', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
);
|
||||
|
||||
# Stat the path/file, ignoring any that are missing
|
||||
my $oStat = $self->info($strFile, {bIgnoreMissing => true});
|
||||
|
||||
# Generate file data if stat succeeded (i.e. file exists)
|
||||
my $hFile;
|
||||
|
||||
if (defined($oStat))
|
||||
{
|
||||
# Check for regular file
|
||||
if (S_ISREG($oStat->mode))
|
||||
{
|
||||
$hFile->{type} = 'f';
|
||||
|
||||
# Get size
|
||||
$hFile->{size} = $oStat->size;
|
||||
|
||||
# Get modification time
|
||||
$hFile->{modification_time} = $oStat->mtime;
|
||||
}
|
||||
# Check for directory
|
||||
elsif (S_ISDIR($oStat->mode))
|
||||
{
|
||||
$hFile->{type} = 'd';
|
||||
}
|
||||
# Check for link
|
||||
elsif (S_ISLNK($oStat->mode))
|
||||
{
|
||||
$hFile->{type} = 'l';
|
||||
$hFile->{link_destination} = $self->linkDestination($strFile);
|
||||
}
|
||||
# Not a recognized type
|
||||
else
|
||||
{
|
||||
confess &log(ERROR, "${strFile} is not of type directory, file, or link", ERROR_FILE_INVALID);
|
||||
}
|
||||
|
||||
# Get user name
|
||||
$hFile->{user} = getpwuid($oStat->uid);
|
||||
|
||||
# Get group name
|
||||
$hFile->{group} = getgrgid($oStat->gid);
|
||||
|
||||
# Get mode
|
||||
if ($hFile->{type} ne 'l')
|
||||
{
|
||||
$hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode));
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'hFile', value => $hFile, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# move - move path/file
|
||||
####################################################################################################################################
|
||||
sub move
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strSourceFile,
|
||||
$strDestinationFile,
|
||||
$bPathCreate,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->move', \@_,
|
||||
{name => 'strSourceFile', trace => true},
|
||||
{name => 'strDestinationFile', trace => true},
|
||||
{name => 'bPathCreate', default => false, trace => true},
|
||||
);
|
||||
|
||||
# Get source and destination paths
|
||||
my $strSourcePathFile = dirname($strSourceFile);
|
||||
my $strDestinationPathFile = dirname($strDestinationFile);
|
||||
|
||||
# Move the file
|
||||
if (!rename($strSourceFile, $strDestinationFile))
|
||||
{
|
||||
my $strMessage = "unable to move '${strSourceFile}'";
|
||||
|
||||
# If something is missing determine if it is the source or destination
|
||||
if ($OS_ERROR{ENOENT})
|
||||
{
|
||||
if (!$self->exists($strSourceFile))
|
||||
{
|
||||
logErrorResult(ERROR_FILE_MISSING, "${strMessage} because it is missing");
|
||||
}
|
||||
|
||||
if ($bPathCreate)
|
||||
{
|
||||
# Attempt to create the path - ignore exists here in case another process creates it first
|
||||
$self->pathCreate($strDestinationPathFile, {bCreateParent => true, bIgnoreExists => true});
|
||||
|
||||
# Try move again
|
||||
$self->move($strSourceFile, $strDestinationFile);
|
||||
}
|
||||
else
|
||||
{
|
||||
logErrorResult(ERROR_PATH_MISSING, "${strMessage} to missing path '${strDestinationPathFile}'");
|
||||
}
|
||||
}
|
||||
# Else raise the error
|
||||
else
|
||||
{
|
||||
logErrorResult(ERROR_FILE_MOVE, "${strMessage} to '${strDestinationFile}'", $OS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# openRead - open file for reading
|
||||
####################################################################################################################################
|
||||
sub openRead
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->openRead', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
my $oFileIO = new pgBackRest::Storage::Posix::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oFileIO', value => $oFileIO, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# openWrite - open file for writing
|
||||
####################################################################################################################################
|
||||
sub openWrite
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
$strMode,
|
||||
$strUser,
|
||||
$strGroup,
|
||||
$lTimestamp,
|
||||
$bPathCreate,
|
||||
$bAtomic,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->openWrite', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
{name => 'strMode', optional => true, trace => true},
|
||||
{name => 'strUser', optional => true, trace => true},
|
||||
{name => 'strGroup', optional => true, trace => true},
|
||||
{name => 'lTimestamp', optional => true, trace => true},
|
||||
{name => 'bPathCreate', optional => true, trace => true},
|
||||
{name => 'bAtomic', optional => true, trace => true},
|
||||
);
|
||||
|
||||
my $oFileIO = new pgBackRest::Storage::Posix::FileWrite(
|
||||
$self, $strFile,
|
||||
{strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,
|
||||
bAtomic => $bAtomic, bSync => $self->{bFileSync}});
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oFileIO', value => $oFileIO, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# owner - change ownership of path/file
|
||||
####################################################################################################################################
|
||||
sub owner
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFilePath,
|
||||
$strUser,
|
||||
$strGroup,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->owner', \@_,
|
||||
{name => 'strFilePath', trace => true},
|
||||
{name => 'strUser', optional => true, trace => true},
|
||||
{name => 'strGroup', optional => true, trace => true},
|
||||
);
|
||||
|
||||
# Only proceed if user or group was specified
|
||||
if (defined($strUser) || defined($strGroup))
|
||||
{
|
||||
my $strMessage = "unable to set ownership for '${strFilePath}'";
|
||||
my $iUserId;
|
||||
my $iGroupId;
|
||||
|
||||
# If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that
|
||||
# both user and group be set.
|
||||
my $oStat = $self->info($strFilePath);
|
||||
|
||||
if (!defined($strUser))
|
||||
{
|
||||
$iUserId = $oStat->uid;
|
||||
}
|
||||
|
||||
if (!defined($strGroup))
|
||||
{
|
||||
$iGroupId = $oStat->gid;
|
||||
}
|
||||
|
||||
# Lookup user if specified
|
||||
if (defined($strUser))
|
||||
{
|
||||
$iUserId = getpwnam($strUser);
|
||||
|
||||
if (!defined($iUserId))
|
||||
{
|
||||
logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist");
|
||||
}
|
||||
}
|
||||
|
||||
# Lookup group if specified
|
||||
if (defined($strGroup))
|
||||
{
|
||||
$iGroupId = getgrnam($strGroup);
|
||||
|
||||
if (!defined($iGroupId))
|
||||
{
|
||||
logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist");
|
||||
}
|
||||
}
|
||||
|
||||
# Set ownership on the file if the user or group would be changed
|
||||
if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid)
|
||||
{
|
||||
if (!chown($iUserId, $iGroupId, $strFilePath))
|
||||
{
|
||||
logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathCreate - create path
|
||||
####################################################################################################################################
|
||||
sub pathCreate
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$strMode,
|
||||
$bIgnoreExists,
|
||||
$bCreateParent,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathCreate', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'strMode', optional => true, default => '0750', trace => true},
|
||||
{name => 'bIgnoreExists', optional => true, default => false, trace => true},
|
||||
{name => 'bCreateParent', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Attempt to create the directory
|
||||
if (!mkdir($strPath, oct($strMode)))
|
||||
{
|
||||
my $strMessage = "unable to create path '${strPath}'";
|
||||
|
||||
# If parent path is missing
|
||||
if ($OS_ERROR{ENOENT})
|
||||
{
|
||||
if (!$bCreateParent)
|
||||
{
|
||||
confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING);
|
||||
}
|
||||
|
||||
# Create parent path
|
||||
$self->pathCreate(dirname($strPath), {strMode => $strMode, bIgnoreExists => true, bCreateParent => $bCreateParent});
|
||||
|
||||
# Create path
|
||||
$self->pathCreate($strPath, {strMode => $strMode, bIgnoreExists => true});
|
||||
}
|
||||
# Else if path already exists
|
||||
elsif ($OS_ERROR{EEXIST})
|
||||
{
|
||||
if (!$bIgnoreExists)
|
||||
{
|
||||
confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathExists - check if path exists
|
||||
####################################################################################################################################
|
||||
sub pathExists
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathExists', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
# Does the path/file exist?
|
||||
my $bExists = true;
|
||||
my $oStat = lstat($strPath);
|
||||
|
||||
# Use stat to test if path exists
|
||||
if (defined($oStat))
|
||||
{
|
||||
# Check that it is actually a path
|
||||
$bExists = S_ISDIR($oStat->mode) ? true : false;
|
||||
}
|
||||
else
|
||||
{
|
||||
# If the error is not entry missing, then throw error
|
||||
if (!$OS_ERROR{ENOENT})
|
||||
{
|
||||
logErrorResult(ERROR_FILE_EXISTS, "unable to test if path '${strPath}' exists", $OS_ERROR);
|
||||
}
|
||||
|
||||
$bExists = false;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bExists', value => $bExists, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathSync - perform fsync on path
|
||||
####################################################################################################################################
|
||||
sub pathSync
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathSync', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
open(my $hPath, "<", $strPath)
|
||||
or confess &log(ERROR, "unable to open '${strPath}' for sync", ERROR_PATH_OPEN);
|
||||
open(my $hPathDup, ">&", $hPath)
|
||||
or confess &log(ERROR, "unable to duplicate '${strPath}' handle for sync", ERROR_PATH_OPEN);
|
||||
|
||||
$hPathDup->sync()
|
||||
or confess &log(ERROR, "unable to sync path '${strPath}'", ERROR_PATH_SYNC);
|
||||
|
||||
close($hPathDup);
|
||||
close($hPath);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# remove - remove path/file
|
||||
####################################################################################################################################
|
||||
sub remove
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPathFile,
|
||||
$bIgnoreMissing,
|
||||
$bRecurse,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->remove', \@_,
|
||||
{name => 'strPathFile', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
{name => 'bRecurse', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Working variables
|
||||
my $bRemoved = true;
|
||||
|
||||
# Remove a tree
|
||||
if ($bRecurse)
|
||||
{
|
||||
# Dynamically load the driver
|
||||
require pgBackRest::LibC;
|
||||
pgBackRest::LibC->import(qw(:storage));
|
||||
|
||||
storagePosixPathRemove($strPathFile, !$bIgnoreMissing, $bRecurse)
|
||||
}
|
||||
# Only remove the specified file
|
||||
else
|
||||
{
|
||||
foreach my $strFile (ref($strPathFile) ? @{$strPathFile} : ($strPathFile))
|
||||
{
|
||||
if (unlink($strFile) != 1)
|
||||
{
|
||||
$bRemoved = false;
|
||||
|
||||
# Throw error if this is not an ignored missing file
|
||||
if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
|
||||
{
|
||||
logErrorResult(
|
||||
$OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to remove file '${strFile}'", $OS_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bRemoved', value => $bRemoved, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters/Setters
|
||||
####################################################################################################################################
|
||||
sub capability {true}
|
||||
sub className {STORAGE_POSIX_DRIVER}
|
||||
sub tempExtension {shift->{strTempExtension}}
|
||||
sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift}
|
||||
|
||||
1;
|
||||
@ -1,105 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Posix File Read
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Posix::FileRead;
|
||||
use parent 'pgBackRest::Common::Io::Handle';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Fcntl qw(O_RDONLY);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oDriver,
|
||||
$strName,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oDriver', trace => true},
|
||||
{name => 'strName', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Open the file
|
||||
my $fhFile;
|
||||
|
||||
if (!sysopen($fhFile, $strName, O_RDONLY))
|
||||
{
|
||||
if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))
|
||||
{
|
||||
logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to open '${strName}'", $OS_ERROR);
|
||||
}
|
||||
|
||||
undef($fhFile);
|
||||
}
|
||||
|
||||
# Create IO object if open succeeded
|
||||
my $self;
|
||||
|
||||
if (defined($fhFile))
|
||||
{
|
||||
# Set file mode to binary
|
||||
binmode($fhFile);
|
||||
|
||||
# Create the class hash
|
||||
$self = $class->SUPER::new("'${strName}'", $fhFile);
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{oDriver} = $oDriver;
|
||||
$self->{strName} = $strName;
|
||||
$self->{fhFile} = $fhFile;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->handle()))
|
||||
{
|
||||
# Close the file
|
||||
close($self->handle());
|
||||
undef($self->{fhFile});
|
||||
|
||||
# Close parent
|
||||
$self->SUPER::close();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub handle {shift->{fhFile}}
|
||||
sub name {shift->{strName}}
|
||||
|
||||
1;
|
||||
@ -1,209 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Posix File Write
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::Posix::FileWrite;
|
||||
use parent 'pgBackRest::Common::Io::Handle';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
use pgBackRest::Common::Io::Handle;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oDriver,
|
||||
$strName,
|
||||
$strMode,
|
||||
$strUser,
|
||||
$strGroup,
|
||||
$lTimestamp,
|
||||
$bPathCreate,
|
||||
$bAtomic,
|
||||
$bSync,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oDriver', trace => true},
|
||||
{name => 'strName', trace => true},
|
||||
{name => 'strMode', optional => true, trace => true},
|
||||
{name => 'strUser', optional => true, trace => true},
|
||||
{name => 'strGroup', optional => true, trace => true},
|
||||
{name => 'lTimestamp', optional => true, trace => true},
|
||||
{name => 'bPathCreate', optional => true, default => false, trace => true},
|
||||
{name => 'bAtomic', optional => true, default => false, trace => true},
|
||||
{name => 'bSync', optional => true, default => true, trace => true},
|
||||
);
|
||||
|
||||
# Create the class hash
|
||||
my $self = $class->SUPER::new("'${strName}'");
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{oDriver} = $oDriver;
|
||||
$self->{strName} = $strName;
|
||||
$self->{strMode} = $strMode;
|
||||
$self->{strUser} = $strUser;
|
||||
$self->{strGroup} = $strGroup;
|
||||
$self->{lTimestamp} = $lTimestamp;
|
||||
$self->{bPathCreate} = $bPathCreate;
|
||||
$self->{bAtomic} = $bAtomic;
|
||||
$self->{bSync} = $bSync;
|
||||
|
||||
# If atomic create temp filename
|
||||
if ($self->{bAtomic})
|
||||
{
|
||||
# Create temp file name
|
||||
$self->{strNameTmp} = "$self->{strName}." . $self->{oDriver}->tempExtension();
|
||||
}
|
||||
|
||||
# Open file on first write to avoid creating extraneous files on error
|
||||
$self->{bOpened} = false;
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# open - open the file
|
||||
####################################################################################################################################
|
||||
sub open
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Get the file name
|
||||
my $strFile = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};
|
||||
|
||||
# Open the file
|
||||
if (!sysopen(
|
||||
$self->{fhFile}, $strFile, O_WRONLY | O_CREAT | O_TRUNC, oct(defined($self->{strMode}) ? $self->{strMode} : '0666')))
|
||||
{
|
||||
# If the path does not exist create it if requested
|
||||
if ($OS_ERROR{ENOENT} && $self->{bPathCreate})
|
||||
{
|
||||
$self->{oDriver}->pathCreate(dirname($strFile), {bIgnoreExists => true, bCreateParent => true});
|
||||
$self->{bPathCreate} = false;
|
||||
return $self->open();
|
||||
}
|
||||
|
||||
logErrorResult($OS_ERROR{ENOENT} ? ERROR_PATH_MISSING : ERROR_FILE_OPEN, "unable to open '${strFile}'", $OS_ERROR);
|
||||
}
|
||||
|
||||
# Set file mode to binary
|
||||
binmode($self->{fhFile});
|
||||
|
||||
# Set the owner
|
||||
$self->{oDriver}->owner($strFile, {strUser => $self->{strUser}, strGroup => $self->{strGroup}});
|
||||
|
||||
# Set handle
|
||||
$self->handleWriteSet($self->{fhFile});
|
||||
|
||||
# Mark file as opened
|
||||
$self->{bOpened} = true;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# write - write data to a file
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
|
||||
# Open file if it is not open already
|
||||
$self->open() if !$self->opened();
|
||||
|
||||
return $self->SUPER::write($rtBuffer);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->handle()))
|
||||
{
|
||||
# Sync the file
|
||||
if ($self->{bSync})
|
||||
{
|
||||
$self->handle()->sync();
|
||||
}
|
||||
|
||||
# Close the file
|
||||
close($self->handle());
|
||||
undef($self->{fhFile});
|
||||
|
||||
# Get current filename
|
||||
my $strCurrentName = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};
|
||||
|
||||
# Set the modification time
|
||||
if (defined($self->{lTimestamp}))
|
||||
{
|
||||
utime(time(), $self->{lTimestamp}, $strCurrentName)
|
||||
or logErrorResult(ERROR_FILE_WRITE, "unable to set time for '${strCurrentName}'", $OS_ERROR);
|
||||
}
|
||||
|
||||
# Move the file from temp to final if atomic
|
||||
if ($self->{bAtomic})
|
||||
{
|
||||
$self->{oDriver}->move($strCurrentName, $self->{strName});
|
||||
}
|
||||
|
||||
# Set result
|
||||
$self->resultSet(COMMON_IO_HANDLE, $self->{lSize});
|
||||
|
||||
# Close parent
|
||||
$self->SUPER::close();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Close the handle if it is open (in case close() was never called)
|
||||
####################################################################################################################################
|
||||
sub DESTROY
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
if (defined($self->handle()))
|
||||
{
|
||||
CORE::close($self->handle());
|
||||
undef($self->{fhFile});
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub handle {shift->{fhFile}}
|
||||
sub opened {shift->{bOpened}}
|
||||
sub name {shift->{strName}}
|
||||
|
||||
1;
|
||||
@ -1,283 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Authentication
|
||||
#
|
||||
# Contains the functions required to do S3 authentication. It's a complicated topic and too much to cover here, but there is
|
||||
# excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html.
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::Auth;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Digest::SHA qw(hmac_sha256 hmac_sha256_hex);
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use POSIX qw(strftime);
|
||||
|
||||
use pgBackRest::Common::Http::Common;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
|
||||
####################################################################################################################################
|
||||
# Constants
|
||||
####################################################################################################################################
|
||||
use constant S3 => 's3';
|
||||
use constant AWS4 => 'AWS4';
|
||||
use constant AWS4_REQUEST => 'aws4_request';
|
||||
use constant AWS4_HMAC_SHA256 => 'AWS4-HMAC-SHA256';
|
||||
|
||||
use constant S3_HEADER_AUTHORIZATION => 'authorization';
|
||||
push @EXPORT, qw(S3_HEADER_AUTHORIZATION);
|
||||
use constant S3_HEADER_DATE => 'x-amz-date';
|
||||
push @EXPORT, qw(S3_HEADER_DATE);
|
||||
use constant S3_HEADER_CONTENT_SHA256 => 'x-amz-content-sha256';
|
||||
push @EXPORT, qw(S3_HEADER_CONTENT_SHA256);
|
||||
use constant S3_HEADER_HOST => 'host';
|
||||
push @EXPORT, qw(S3_HEADER_HOST);
|
||||
use constant S3_HEADER_TOKEN => 'x-amz-security-token';
|
||||
push @EXPORT, qw(S3_HEADER_TOKEN);
|
||||
|
||||
use constant PAYLOAD_DEFAULT_HASH => cryptoHashOne('sha256', '');
|
||||
push @EXPORT, qw(PAYLOAD_DEFAULT_HASH);
|
||||
|
||||
####################################################################################################################################
|
||||
# s3DateTime - format date/time for authentication
|
||||
####################################################################################################################################
|
||||
sub s3DateTime
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$lTime,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::s3DateTime', \@_,
|
||||
{name => 'lTime', default => time(), trace => true},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strDateTime', value => strftime("%Y%m%dT%H%M%SZ", gmtime($lTime)), trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(s3DateTime);
|
||||
|
||||
####################################################################################################################################
|
||||
# s3CanonicalRequest - strictly formatted version of the HTTP request used for signing
|
||||
####################################################################################################################################
|
||||
sub s3CanonicalRequest
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strVerb,
|
||||
$strUri,
|
||||
$strQuery,
|
||||
$hHeader,
|
||||
$strPayloadHash,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::s3CanonicalRequest', \@_,
|
||||
{name => 'strVerb', trace => true},
|
||||
{name => 'strUri', trace => true},
|
||||
{name => 'strQuery', trace => true},
|
||||
{name => 'hHeader', trace => true},
|
||||
{name => 'strPayloadHash', trace => true},
|
||||
);
|
||||
|
||||
# Create the canonical request
|
||||
my $strCanonicalRequest =
|
||||
"${strVerb}\n${strUri}\n${strQuery}\n";
|
||||
my $strSignedHeaders;
|
||||
|
||||
foreach my $strHeader (sort(keys(%{$hHeader})))
|
||||
{
|
||||
if (lc($strHeader) ne $strHeader)
|
||||
{
|
||||
confess &log(ASSERT, "header '${strHeader}' must be lower case");
|
||||
}
|
||||
|
||||
$strCanonicalRequest .= $strHeader . ":$hHeader->{$strHeader}\n";
|
||||
$strSignedHeaders .= (defined($strSignedHeaders) ? qw(;) : '') . lc($strHeader);
|
||||
}
|
||||
|
||||
$strCanonicalRequest .= "\n${strSignedHeaders}\n${strPayloadHash}";
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true},
|
||||
{name => 'strSignedHeaders', value => $strSignedHeaders, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(s3CanonicalRequest);
|
||||
|
||||
####################################################################################################################################
|
||||
# s3SigningKey - signing keys last for seven days, but we'll regenerate every day because it doesn't seem too burdensome
|
||||
####################################################################################################################################
|
||||
my $hSigningKeyCache; # Cache signing keys rather than regenerating them every time
|
||||
|
||||
sub s3SigningKey
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strDate,
|
||||
$strRegion,
|
||||
$strSecretAccessKey,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::s3SigningKey', \@_,
|
||||
{name => 'strDate', trace => true},
|
||||
{name => 'strRegion', trace => true},
|
||||
{name => 'strSecretAccessKey', redact => true, trace => true},
|
||||
);
|
||||
|
||||
# Check for signing key in cache
|
||||
my $strSigningKey = $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey};
|
||||
|
||||
# If not found then generate it
|
||||
if (!defined($strSigningKey))
|
||||
{
|
||||
my $strDateKey = hmac_sha256($strDate, AWS4 . $strSecretAccessKey);
|
||||
my $strRegionKey = hmac_sha256($strRegion, $strDateKey);
|
||||
my $strServiceKey = hmac_sha256(S3, $strRegionKey);
|
||||
$strSigningKey = hmac_sha256(AWS4_REQUEST, $strServiceKey);
|
||||
|
||||
# Cache the signing key
|
||||
$hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey} = $strSigningKey;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strSigningKey', value => $strSigningKey, redact => true, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(s3SigningKey);
|
||||
|
||||
####################################################################################################################################
|
||||
# s3StringToSign - string that will be signed by the signing key for authentication
|
||||
####################################################################################################################################
|
||||
sub s3StringToSign
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strDateTime,
|
||||
$strRegion,
|
||||
$strCanonicalRequestHash,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::s3StringToSign', \@_,
|
||||
{name => 'strDateTime', trace => true},
|
||||
{name => 'strRegion', trace => true},
|
||||
{name => 'strCanonicalRequestHash', trace => true},
|
||||
);
|
||||
|
||||
my $strStringToSign =
|
||||
AWS4_HMAC_SHA256 . "\n${strDateTime}\n" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . '/' . AWS4_REQUEST . "\n" .
|
||||
$strCanonicalRequestHash;
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'strStringToSign', value => $strStringToSign, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(s3StringToSign);
|
||||
|
||||
####################################################################################################################################
|
||||
# s3AuthorizationHeader - authorization string that will be used in the HTTP "authorization" header
|
||||
####################################################################################################################################
|
||||
sub s3AuthorizationHeader
|
||||
{
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strRegion,
|
||||
$strHost,
|
||||
$strVerb,
|
||||
$strUri,
|
||||
$strQuery,
|
||||
$strDateTime,
|
||||
$hHeader,
|
||||
$strAccessKeyId,
|
||||
$strSecretAccessKey,
|
||||
$strSecurityToken,
|
||||
$strPayloadHash,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '::s3AuthorizationHeader', \@_,
|
||||
{name => 'strRegion', trace => true},
|
||||
{name => 'strHost', trace => true},
|
||||
{name => 'strVerb', trace => true},
|
||||
{name => 'strUri', trace => true},
|
||||
{name => 'strQuery', trace => true},
|
||||
{name => 'strDateTime', trace => true},
|
||||
{name => 'hHeader', required => false, trace => true},
|
||||
{name => 'strAccessKeyId', redact => true, trace => true},
|
||||
{name => 'strSecretAccessKey', redact => true, trace => true},
|
||||
{name => 'strSecurityToken', required => false, redact => true, trace => true},
|
||||
{name => 'strPayloadHash', trace => true},
|
||||
);
|
||||
|
||||
# Delete the authorization header if it already exists. This could happen on a retry.
|
||||
delete($hHeader->{&S3_HEADER_AUTHORIZATION});
|
||||
|
||||
# Add s3 required headers
|
||||
$hHeader->{&S3_HEADER_HOST} = $strHost;
|
||||
$hHeader->{&S3_HEADER_CONTENT_SHA256} = $strPayloadHash;
|
||||
$hHeader->{&S3_HEADER_DATE} = $strDateTime;
|
||||
|
||||
# Add security token if defined
|
||||
if (defined($strSecurityToken))
|
||||
{
|
||||
$hHeader->{&S3_HEADER_TOKEN} = $strSecurityToken;
|
||||
}
|
||||
|
||||
# Create authorization string
|
||||
my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest(
|
||||
$strVerb, httpUriEncode($strUri, true), $strQuery, $hHeader, $strPayloadHash);
|
||||
my $strStringToSign = s3StringToSign($strDateTime, $strRegion, cryptoHashOne('sha256', $strCanonicalRequest));
|
||||
|
||||
$hHeader->{&S3_HEADER_AUTHORIZATION} =
|
||||
AWS4_HMAC_SHA256 . " Credential=${strAccessKeyId}/" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . qw(/) .
|
||||
AWS4_REQUEST . ",SignedHeaders=${strSignedHeaders},Signature=" . hmac_sha256_hex($strStringToSign,
|
||||
s3SigningKey(substr($strDateTime, 0, 8), $strRegion, $strSecretAccessKey));
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'hHeader', value => $hHeader, trace => true},
|
||||
{name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true},
|
||||
{name => 'strSignedHeaders', value => $strSignedHeaders, trace => true},
|
||||
{name => 'strStringToSign', value => $strStringToSign, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(s3AuthorizationHeader);
|
||||
|
||||
1;
|
||||
@ -1,507 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Storage
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::Driver;
|
||||
use parent 'pgBackRest::Storage::S3::Request';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use Digest::MD5 qw(md5_base64);
|
||||
use File::Basename qw(basename dirname);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Common::Xml;
|
||||
use pgBackRest::Storage::S3::FileRead;
|
||||
use pgBackRest::Storage::S3::FileWrite;
|
||||
use pgBackRest::Storage::S3::Request;
|
||||
use pgBackRest::Storage::S3::Info;
|
||||
|
||||
####################################################################################################################################
|
||||
# Package name constant
|
||||
####################################################################################################################################
|
||||
use constant STORAGE_S3_DRIVER => __PACKAGE__;
|
||||
push @EXPORT, qw(STORAGE_S3_DRIVER);
|
||||
|
||||
####################################################################################################################################
|
||||
# Query constants
|
||||
####################################################################################################################################
|
||||
use constant S3_QUERY_CONTINUATION_TOKEN => 'continuation-token';
|
||||
use constant S3_QUERY_DELIMITER => 'delimiter';
|
||||
use constant S3_QUERY_LIST_TYPE => 'list-type';
|
||||
use constant S3_QUERY_PREFIX => 'prefix';
|
||||
|
||||
####################################################################################################################################
|
||||
# Batch maximum size
|
||||
####################################################################################################################################
|
||||
use constant S3_BATCH_MAX => 1000;
|
||||
|
||||
####################################################################################################################################
|
||||
# openWrite - open a file for write
|
||||
####################################################################################################################################
|
||||
sub openWrite
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->openWrite', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
);
|
||||
|
||||
my $oFileIO = new pgBackRest::Storage::S3::FileWrite($self, $strFile);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oFileIO', value => $oFileIO, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# openRead
|
||||
####################################################################################################################################
|
||||
sub openRead
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->openRead', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
my $oFileIO = new pgBackRest::Storage::S3::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oFileIO', value => $oFileIO, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# manifest
|
||||
####################################################################################################################################
|
||||
sub manifest
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$bRecurse,
|
||||
$bPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->manifest', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
# Optional parameters not part of the driver spec
|
||||
{name => 'bRecurse', optional => true, default => true, trace => true},
|
||||
{name => 'bPath', optional => true, default => true, trace => true},
|
||||
);
|
||||
|
||||
# Determine the prefix (this is the search path within the bucket)
|
||||
my $strPrefix = $strPath eq qw{/} ? undef : substr($strPath, 1) . ($bPath ? qw{/} : '');
|
||||
|
||||
# A delimiter must be used if recursion is not desired
|
||||
my $strDelimiter = $bRecurse ? undef : '/';
|
||||
|
||||
# Hash to hold the manifest
|
||||
my $hManifest = {};
|
||||
|
||||
# Continuation token - returned from requests where there is more data to be fetched
|
||||
my $strContinuationToken;
|
||||
|
||||
do
|
||||
{
|
||||
# Get the file list
|
||||
my $oResponse = $self->request(
|
||||
HTTP_VERB_GET, {hQuery =>
|
||||
{&S3_QUERY_LIST_TYPE => 2, &S3_QUERY_PREFIX => $strPrefix, &S3_QUERY_DELIMITER => $strDelimiter,
|
||||
&S3_QUERY_CONTINUATION_TOKEN => $strContinuationToken}, strResponseType => S3_RESPONSE_TYPE_XML});
|
||||
|
||||
# Modify the prefix for file searches so the filename is not stripped off
|
||||
if (defined($strPrefix) && !$bPath)
|
||||
{
|
||||
# If there are no paths in the prefix then undef it
|
||||
if (index($strPrefix, qw{/}) == -1)
|
||||
{
|
||||
undef($strPrefix);
|
||||
}
|
||||
else
|
||||
{
|
||||
$strPrefix = dirname($strPrefix) . qw{/};
|
||||
}
|
||||
}
|
||||
|
||||
# Store files
|
||||
foreach my $oFile (xmlTagChildren($oResponse, "Contents"))
|
||||
{
|
||||
my $strName = xmlTagText($oFile, "Key");
|
||||
|
||||
# Strip off prefix
|
||||
if (defined($strPrefix))
|
||||
{
|
||||
$strName = substr($strName, length($strPrefix));
|
||||
}
|
||||
|
||||
$hManifest->{$strName}->{type} = 'f';
|
||||
$hManifest->{$strName}->{size} = xmlTagText($oFile, 'Size') + 0;
|
||||
|
||||
# Generate paths from the name if recursing
|
||||
if ($bRecurse)
|
||||
{
|
||||
my @stryName = split(qw{/}, $strName);
|
||||
|
||||
if (@stryName > 1)
|
||||
{
|
||||
$strName = undef;
|
||||
|
||||
for (my $iIndex = 0; $iIndex < @stryName - 1; $iIndex++)
|
||||
{
|
||||
$strName .= (defined($strName) ? qw{/} : '') . $stryName[$iIndex];
|
||||
$hManifest->{$strName}->{type} = 'd';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Store directories
|
||||
if ($bPath && !$bRecurse)
|
||||
{
|
||||
foreach my $oPath (xmlTagChildren($oResponse, "CommonPrefixes"))
|
||||
{
|
||||
my $strName = xmlTagText($oPath, "Prefix");
|
||||
|
||||
# Strip off prefix
|
||||
if (defined($strPrefix))
|
||||
{
|
||||
$strName = substr($strName, length($strPrefix));
|
||||
}
|
||||
|
||||
# Strip off final /
|
||||
$strName = substr($strName, 0, length($strName) - 1);
|
||||
|
||||
$hManifest->{$strName}->{type} = 'd';
|
||||
}
|
||||
}
|
||||
|
||||
$strContinuationToken = xmlTagText($oResponse, "NextContinuationToken", false);
|
||||
}
|
||||
while (defined($strContinuationToken));
|
||||
|
||||
# Add . for the initial path (this is just for compatibility with filesystems that have directories)
|
||||
if ($bPath)
|
||||
{
|
||||
$hManifest->{qw{.}}->{type} = 'd';
|
||||
}
|
||||
|
||||
# use Data::Dumper; &log(WARN, 'MANIFEST' . Dumper($hManifest));
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'hManifest', value => $hManifest, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# list - list a directory
|
||||
####################################################################################################################################
|
||||
sub list
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
$strExpression
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->list', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
{name => 'strExpression', optional => true, trace => true},
|
||||
);
|
||||
|
||||
# Use the regexp to build a prefix to shorten searches
|
||||
my $strPrefix = regexPrefix($strExpression);
|
||||
|
||||
# Get list using manifest function
|
||||
my @stryFileList = grep(
|
||||
!/^\.$/i, keys(%{$self->manifest(
|
||||
$strPath . (defined($strPrefix) ? "/${strPrefix}" : ''), {bRecurse => false, bPath => !defined($strPrefix)})}));
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'stryFileList', value => \@stryFileList, ref => true, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathCreate - directories do no exist in s3 so this is a noop
|
||||
####################################################################################################################################
|
||||
sub pathCreate
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathCreate', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathSync - directories do not exist in s3 so this is a noop
|
||||
####################################################################################################################################
|
||||
sub pathSync
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathSync', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn($strOperation);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# exists - check if a file exists
|
||||
####################################################################################################################################
|
||||
sub exists
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->exists', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
);
|
||||
|
||||
# Does the path/file exist?
|
||||
my $bExists = defined($self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}) ? true : false;
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bExists', value => $bExists, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# pathExists
|
||||
####################################################################################################################################
|
||||
sub pathExists
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPath,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->pathExists', \@_,
|
||||
{name => 'strPath', trace => true},
|
||||
);
|
||||
|
||||
my $bExists = true;
|
||||
|
||||
# Only check if path <> /
|
||||
if ($strPath ne qw{/})
|
||||
{
|
||||
# Does the path exist?
|
||||
my $rhInfo = $self->manifest(dirname($strPath), {bRecurse => false, bPath => true})->{basename($strPath)};
|
||||
$bExists = defined($rhInfo) && $rhInfo->{type} eq 'd' ? true : false;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bExists', value => $bExists, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# info - get information about a file
|
||||
####################################################################################################################################
|
||||
sub info
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strFile,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->info', \@_,
|
||||
{name => 'strFile', trace => true},
|
||||
);
|
||||
|
||||
# Get the file
|
||||
my $rhFile = $self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)};
|
||||
|
||||
if (!defined($rhFile))
|
||||
{
|
||||
confess &log(ERROR, "unable to get info for missing file ${strFile}", ERROR_FILE_MISSING);
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oInfo', value => new pgBackRest::Storage::S3::Info($rhFile->{size}), trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# remove
|
||||
####################################################################################################################################
|
||||
sub remove
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$rstryFile,
|
||||
$bRecurse,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->remove', \@_,
|
||||
{name => 'rstryFile', trace => true},
|
||||
{name => 'bRecurse', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Remove a tree
|
||||
if ($bRecurse)
|
||||
{
|
||||
my $rhManifest = $self->manifest($rstryFile);
|
||||
my @stryRemoveFile;
|
||||
|
||||
# Iterate all files in the manifest
|
||||
foreach my $strFile (sort({$b cmp $a} keys(%{$rhManifest})))
|
||||
{
|
||||
next if $rhManifest->{$strFile}->{type} eq 'd';
|
||||
push(@stryRemoveFile, "${rstryFile}/${strFile}");
|
||||
}
|
||||
|
||||
# Remove files
|
||||
if (@stryRemoveFile > 0)
|
||||
{
|
||||
$self->remove(\@stryRemoveFile);
|
||||
}
|
||||
}
|
||||
# Only remove the specified file
|
||||
else
|
||||
{
|
||||
# If stryFile is a scalar, convert to an array
|
||||
my $rstryFileAll = ref($rstryFile) ? $rstryFile : [$rstryFile];
|
||||
|
||||
do
|
||||
{
|
||||
my $strFile = shift(@{$rstryFileAll});
|
||||
my $iTotal = 0;
|
||||
my $strXml = XML_HEADER . '<Delete><Quiet>true</Quiet>';
|
||||
|
||||
while (defined($strFile))
|
||||
{
|
||||
$iTotal++;
|
||||
$strXml .= '<Object><Key>' . xmlFromText(substr($strFile, 1)) . '</Key></Object>';
|
||||
|
||||
$strFile = $iTotal < S3_BATCH_MAX ? shift(@{$rstryFileAll}) : undef;
|
||||
}
|
||||
|
||||
$strXml .= '</Delete>';
|
||||
|
||||
my $hHeader = {'content-md5' => md5_base64($strXml) . '=='};
|
||||
|
||||
# Delete a file
|
||||
my $oResponse = $self->request(
|
||||
HTTP_VERB_POST,
|
||||
{hQuery => 'delete=', rstrBody => \$strXml, hHeader => $hHeader, strResponseType => S3_RESPONSE_TYPE_XML});
|
||||
}
|
||||
while (@{$rstryFileAll} > 0);
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => true, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub capability {false}
|
||||
sub className {STORAGE_S3_DRIVER}
|
||||
|
||||
1;
|
||||
@ -1,68 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 File Read
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::FileRead;
|
||||
use parent 'pgBackRest::Common::Http::Client';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Digest::MD5 qw(md5_base64);
|
||||
use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Xml;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::S3::Request;
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oDriver,
|
||||
$strName,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oDriver', trace => true},
|
||||
{name => 'strName', trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Open file
|
||||
my $self = $oDriver->request(
|
||||
HTTP_VERB_GET, {strUri => $strName, strResponseType => S3_RESPONSE_TYPE_IO, bIgnoreMissing => $bIgnoreMissing});
|
||||
|
||||
# Bless with new class if file exists
|
||||
if (defined($self))
|
||||
{
|
||||
bless $self, $class;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub name {shift->{strName}}
|
||||
|
||||
1;
|
||||
@ -1,205 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 File Write
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::FileWrite;
|
||||
use parent 'pgBackRest::Common::Io::Base';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Digest::MD5 qw(md5_base64);
|
||||
use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Io::Handle;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Xml;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::S3::Request;
|
||||
|
||||
####################################################################################################################################
|
||||
# Constants
|
||||
####################################################################################################################################
|
||||
use constant S3_BUFFER_MAX => 16777216;
|
||||
|
||||
####################################################################################################################################
|
||||
# CONSTRUCTOR
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$oDriver,
|
||||
$strName,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oDriver', trace => true},
|
||||
{name => 'strName', trace => true},
|
||||
);
|
||||
|
||||
# Create the class hash
|
||||
my $self = $class->SUPER::new("'${strName}'");
|
||||
bless $self, $class;
|
||||
|
||||
# Set variables
|
||||
$self->{oDriver} = $oDriver;
|
||||
$self->{strName} = $strName;
|
||||
|
||||
# Start with an empty buffer
|
||||
$self->{rtBuffer} = '';
|
||||
|
||||
# Has anything been written?
|
||||
$self->{bWritten} = false;
|
||||
|
||||
# How much has been written?
|
||||
$self->{lSize} = 0;
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# open - open the file
|
||||
####################################################################################################################################
|
||||
sub open
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Request an upload id
|
||||
my $oResponse = $self->{oDriver}->request(
|
||||
HTTP_VERB_POST, {strUri => $self->{strName}, hQuery => 'uploads=', strResponseType => S3_RESPONSE_TYPE_XML});
|
||||
|
||||
$self->{strUploadId} = xmlTagText($oResponse, 'UploadId');
|
||||
|
||||
# Intialize the multi-part array
|
||||
$self->{rstryMultiPart} = [];
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# write - write data to a file
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
my $rtBuffer = shift;
|
||||
|
||||
# Note that write has been called
|
||||
$self->{bWritten} = true;
|
||||
|
||||
if (defined($rtBuffer))
|
||||
{
|
||||
$self->{rtBuffer} .= $$rtBuffer;
|
||||
$self->{lSize} += length($$rtBuffer);
|
||||
|
||||
# Wait until buffer is at least max before writing to avoid writing smaller files multi-part
|
||||
if (length($self->{rtBuffer}) >= S3_BUFFER_MAX)
|
||||
{
|
||||
$self->flush();
|
||||
}
|
||||
|
||||
return length($$rtBuffer);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# flush - flush whatever is in the buffer out
|
||||
####################################################################################################################################
|
||||
sub flush
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Open file if it is not open already
|
||||
$self->open() if !$self->opened();
|
||||
|
||||
# Put a file
|
||||
$self->{oDriver}->request(
|
||||
HTTP_VERB_PUT,
|
||||
{strUri => $self->{strName},
|
||||
hQuery => {'partNumber' => @{$self->{rstryMultiPart}} + 1, 'uploadId' => $self->{strUploadId}},
|
||||
rstrBody => \$self->{rtBuffer}, hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});
|
||||
|
||||
# Store the returned etag
|
||||
push(@{$self->{rstryMultiPart}}, $self->{oDriver}->{hResponseHeader}{&S3_HEADER_ETAG});
|
||||
|
||||
# Clear the buffer
|
||||
$self->{rtBuffer} = '';
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# close - close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Only close if something was written
|
||||
if ($self->{bWritten})
|
||||
{
|
||||
# Make sure close does not run again
|
||||
$self->{bWritten} = false;
|
||||
|
||||
# If the file is open then multipart transfer has already started and must be completed
|
||||
if ($self->opened())
|
||||
{
|
||||
# flush out whatever is in the buffer
|
||||
$self->flush();
|
||||
|
||||
my $strXml = XML_HEADER . '<CompleteMultipartUpload>';
|
||||
my $iPartNo = 0;
|
||||
|
||||
foreach my $strETag (@{$self->{rstryMultiPart}})
|
||||
{
|
||||
$iPartNo++;
|
||||
|
||||
$strXml .= "<Part><PartNumber>${iPartNo}</PartNumber><ETag>${strETag}</ETag></Part>";
|
||||
}
|
||||
|
||||
$strXml .= '</CompleteMultipartUpload>';
|
||||
|
||||
# Finalize file
|
||||
my $oResponse = $self->{oDriver}->request(
|
||||
HTTP_VERB_POST,
|
||||
{strUri => $self->{strName}, hQuery => {'uploadId' => $self->{strUploadId}},
|
||||
rstrBody => \$strXml, hHeader => {'content-md5' => md5_base64($strXml) . '=='},
|
||||
strResponseType => S3_RESPONSE_TYPE_XML});
|
||||
}
|
||||
# Else the file can be transmitted in one block
|
||||
else
|
||||
{
|
||||
$self->{oDriver}->request(
|
||||
HTTP_VERB_PUT,
|
||||
{strUri => $self->{strName}, rstrBody => \$self->{rtBuffer},
|
||||
hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});
|
||||
}
|
||||
}
|
||||
|
||||
# This is how we report the write size back to the caller. It's a bit hokey -- results are based on the object name and in all
|
||||
# other cases I/O passes through Io::Handle. It doesn't seem worth fixing since it could break things and this code is being
|
||||
# migrated to C anyway.
|
||||
$self->resultSet(COMMON_IO_HANDLE, $self->{lSize});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub opened {defined(shift->{strUploadId})}
|
||||
sub name {shift->{strName}}
|
||||
|
||||
1;
|
||||
@ -1,48 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 File Info
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::Info;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Create the class hash
|
||||
my $self = {};
|
||||
bless $self, $class;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
(
|
||||
my $strOperation,
|
||||
$self->{lSize},
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'lSize'},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub size {shift->{lSize}}
|
||||
|
||||
1;
|
||||
@ -1,264 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Request
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::S3::Request;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use IO::Socket::SSL;
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Http::Client;
|
||||
use pgBackRest::Common::Http::Common;
|
||||
use pgBackRest::Common::Io::Base;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Common::Xml;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::S3::Auth;
|
||||
|
||||
####################################################################################################################################
|
||||
# Constants
|
||||
####################################################################################################################################
|
||||
use constant HTTP_VERB_GET => 'GET';
|
||||
push @EXPORT, qw(HTTP_VERB_GET);
|
||||
use constant HTTP_VERB_POST => 'POST';
|
||||
push @EXPORT, qw(HTTP_VERB_POST);
|
||||
use constant HTTP_VERB_PUT => 'PUT';
|
||||
push @EXPORT, qw(HTTP_VERB_PUT);
|
||||
|
||||
use constant S3_HEADER_CONTENT_LENGTH => 'content-length';
|
||||
push @EXPORT, qw(S3_HEADER_CONTENT_LENGTH);
|
||||
use constant S3_HEADER_TRANSFER_ENCODING => 'transfer-encoding';
|
||||
push @EXPORT, qw(S3_HEADER_TRANSFER_ENCODING);
|
||||
use constant S3_HEADER_ETAG => 'etag';
|
||||
push @EXPORT, qw(S3_HEADER_ETAG);
|
||||
|
||||
use constant S3_RESPONSE_TYPE_IO => 'io';
|
||||
push @EXPORT, qw(S3_RESPONSE_TYPE_IO);
|
||||
use constant S3_RESPONSE_TYPE_NONE => 'none';
|
||||
push @EXPORT, qw(S3_RESPONSE_TYPE_NONE);
|
||||
use constant S3_RESPONSE_TYPE_XML => 'xml';
|
||||
push @EXPORT, qw(S3_RESPONSE_TYPE_XML);
|
||||
|
||||
use constant S3_RESPONSE_CODE_SUCCESS => 200;
|
||||
use constant S3_RESPONSE_CODE_ERROR_AUTH => 403;
|
||||
use constant S3_RESPONSE_CODE_ERROR_NOT_FOUND => 404;
|
||||
use constant S3_RESPONSE_CODE_ERROR_RETRY_CLASS => 5;
|
||||
|
||||
use constant S3_RETRY_MAX => 4;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Create the class hash
|
||||
my $self = {};
|
||||
bless $self, $class;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
(
|
||||
my $strOperation,
|
||||
$self->{strBucket},
|
||||
$self->{strEndPoint},
|
||||
$self->{strRegion},
|
||||
$self->{strAccessKeyId},
|
||||
$self->{strSecretAccessKey},
|
||||
$self->{strSecurityToken},
|
||||
$self->{strHost},
|
||||
$self->{iPort},
|
||||
$self->{bVerifySsl},
|
||||
$self->{strCaPath},
|
||||
$self->{strCaFile},
|
||||
$self->{lBufferMax},
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'strBucket'},
|
||||
{name => 'strEndPoint'},
|
||||
{name => 'strRegion'},
|
||||
{name => 'strAccessKeyId', redact => true},
|
||||
{name => 'strSecretAccessKey', redact => true},
|
||||
{name => 'strSecurityToken', optional => true, redact => true},
|
||||
{name => 'strHost', optional => true},
|
||||
{name => 'iPort', optional => true},
|
||||
{name => 'bVerifySsl', optional => true, default => true},
|
||||
{name => 'strCaPath', optional => true},
|
||||
{name => 'strCaFile', optional => true},
|
||||
{name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX},
|
||||
);
|
||||
|
||||
# If host is not set then it will be bucket + endpoint
|
||||
$self->{strHost} = defined($self->{strHost}) ? $self->{strHost} : "$self->{strBucket}.$self->{strEndPoint}";
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self, trace => true}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# request - send a request to S3
|
||||
####################################################################################################################################
|
||||
sub request
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strVerb,
|
||||
$strUri,
|
||||
$hQuery,
|
||||
$hHeader,
|
||||
$rstrBody,
|
||||
$strResponseType,
|
||||
$bIgnoreMissing,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->request', \@_,
|
||||
{name => 'strVerb', trace => true},
|
||||
{name => 'strUri', optional => true, default => '/', trace => true},
|
||||
{name => 'hQuery', optional => true, trace => true},
|
||||
{name => 'hHeader', optional => true, trace => true},
|
||||
{name => 'rstrBody', optional => true, trace => true},
|
||||
{name => 'strResponseType', optional => true, default => S3_RESPONSE_TYPE_NONE, trace => true},
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Server response
|
||||
my $oResponse;
|
||||
|
||||
# Allow retries on S3 internal failures
|
||||
my $bRetry;
|
||||
my $iRetryTotal = 0;
|
||||
|
||||
do
|
||||
{
|
||||
# Assume that a retry will not be attempted which is true in most cases
|
||||
$bRetry = false;
|
||||
|
||||
# Set content length and hash
|
||||
$hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? cryptoHashOne('sha256', $$rstrBody) : PAYLOAD_DEFAULT_HASH;
|
||||
$hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0;
|
||||
|
||||
# Generate authorization header
|
||||
($hHeader, my $strCanonicalRequest, my $strSignedHeaders, my $strStringToSign) = s3AuthorizationHeader(
|
||||
$self->{strRegion}, "$self->{strBucket}.$self->{strEndPoint}", $strVerb, $strUri, httpQuery($hQuery), s3DateTime(),
|
||||
$hHeader, $self->{strAccessKeyId}, $self->{strSecretAccessKey}, $self->{strSecurityToken},
|
||||
$hHeader->{&S3_HEADER_CONTENT_SHA256});
|
||||
|
||||
# Send the request
|
||||
my $oHttpClient = new pgBackRest::Common::Http::Client(
|
||||
$self->{strHost}, $strVerb,
|
||||
{iPort => $self->{iPort}, strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader,
|
||||
rstrRequestBody => $rstrBody, bVerifySsl => $self->{bVerifySsl}, strCaPath => $self->{strCaPath},
|
||||
strCaFile => $self->{strCaFile}, bResponseBodyPrefetch => $strResponseType eq S3_RESPONSE_TYPE_XML,
|
||||
lBufferMax => $self->{lBufferMax}});
|
||||
|
||||
# Check response code
|
||||
my $iResponseCode = $oHttpClient->responseCode();
|
||||
|
||||
if ($iResponseCode == S3_RESPONSE_CODE_SUCCESS)
|
||||
{
|
||||
# Save the response headers locally
|
||||
$self->{hResponseHeader} = $oHttpClient->responseHeader();
|
||||
|
||||
# XML response is expected
|
||||
if ($strResponseType eq S3_RESPONSE_TYPE_XML)
|
||||
{
|
||||
my $rtResponseBody = $oHttpClient->responseBody();
|
||||
|
||||
if ($oHttpClient->contentLength() == 0 || !defined($$rtResponseBody))
|
||||
{
|
||||
confess &log(ERROR,
|
||||
"response type '${strResponseType}' was requested but content length is zero or content is missing",
|
||||
ERROR_PROTOCOL);
|
||||
}
|
||||
|
||||
$oResponse = xmlParse($$rtResponseBody);
|
||||
}
|
||||
# An IO object is expected for file responses
|
||||
elsif ($strResponseType eq S3_RESPONSE_TYPE_IO)
|
||||
{
|
||||
$oResponse = $oHttpClient;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
# If file was not found
|
||||
if ($iResponseCode == S3_RESPONSE_CODE_ERROR_NOT_FOUND)
|
||||
{
|
||||
# If missing files should not be ignored then error
|
||||
if (!$bIgnoreMissing)
|
||||
{
|
||||
confess &log(ERROR, "unable to open '${strUri}': No such file or directory", ERROR_FILE_MISSING);
|
||||
}
|
||||
|
||||
$bRetry = false;
|
||||
}
|
||||
# Else a more serious error
|
||||
else
|
||||
{
|
||||
# Retry for S3 internal or rate-limiting errors (any 5xx error should be retried)
|
||||
if (int($iResponseCode / 100) == S3_RESPONSE_CODE_ERROR_RETRY_CLASS)
|
||||
{
|
||||
# Increment retry total and check if retry should be attempted
|
||||
$iRetryTotal++;
|
||||
$bRetry = $iRetryTotal <= S3_RETRY_MAX;
|
||||
|
||||
# Sleep after first retry just in case data needs to stabilize
|
||||
if ($iRetryTotal > 1)
|
||||
{
|
||||
sleep(5);
|
||||
}
|
||||
}
|
||||
|
||||
# If no retry then throw the error
|
||||
if (!$bRetry)
|
||||
{
|
||||
my $rstrResponseBody = $oHttpClient->responseBody();
|
||||
|
||||
# Redact authorization header because it contains the access key
|
||||
my $strRequestHeader = $oHttpClient->requestHeaderText();
|
||||
$strRequestHeader =~ s/^${\S3_HEADER_AUTHORIZATION}:.*$/${\S3_HEADER_AUTHORIZATION}: <redacted>/mg;
|
||||
|
||||
confess &log(ERROR,
|
||||
'S3 request error' . ($iRetryTotal > 0 ? " after " . (S3_RETRY_MAX + 1) . " tries" : '') .
|
||||
" [$iResponseCode] " . $oHttpClient->responseMessage() .
|
||||
"\n*** request header ***\n${strRequestHeader}" .
|
||||
($iResponseCode == S3_RESPONSE_CODE_ERROR_AUTH ?
|
||||
"\n*** canonical request ***\n" . $strCanonicalRequest .
|
||||
"\n*** signed headers ***\n" . $strSignedHeaders .
|
||||
"\n*** string to sign ***\n" . $strStringToSign : '') .
|
||||
"\n*** response header ***\n" . $oHttpClient->responseHeaderText() .
|
||||
(defined($$rstrResponseBody) ? "\n*** response body ***\n${$rstrResponseBody}" : ''),
|
||||
ERROR_PROTOCOL);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
while ($bRetry);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'oResponse', value => $oResponse, trace => true, ref => true}
|
||||
);
|
||||
}
|
||||
|
||||
1;
|
||||
File diff suppressed because it is too large
Load Diff
190
lib/pgBackRest/Storage/StorageRead.pm
Normal file
190
lib/pgBackRest/Storage/StorageRead.pm
Normal file
@ -0,0 +1,190 @@
|
||||
####################################################################################################################################
|
||||
# C Storage Read Interface
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::StorageRead;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
use Fcntl qw(:mode);
|
||||
use File::stat qw{lstat};
|
||||
use JSON::PP;
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Create the class hash
|
||||
my $self = {};
|
||||
bless $self, $class;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
(
|
||||
my $strOperation,
|
||||
$self->{oStorage},
|
||||
$self->{oStorageCRead},
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oStorage'},
|
||||
{name => 'oStorageCRead'},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Open the file
|
||||
####################################################################################################################################
|
||||
sub open
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => $self->{oStorageCRead}->open() ? true : false, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Read data
|
||||
####################################################################################################################################
|
||||
sub read
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my (
|
||||
$strOperation,
|
||||
$rtBuffer,
|
||||
$iSize,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->read', \@_,
|
||||
{name => 'rtBuffer'},
|
||||
{name => 'iSize'},
|
||||
);
|
||||
|
||||
# Read if not eof
|
||||
my $iActualSize = 0;
|
||||
|
||||
if (!$self->eof())
|
||||
{
|
||||
my $tBuffer = $self->{oStorageCRead}->read($iSize);
|
||||
$iActualSize = length($tBuffer);
|
||||
$$rtBuffer .= $tBuffer;
|
||||
}
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'iActualSize', value => $iActualSize}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Is the file at eof?
|
||||
####################################################################################################################################
|
||||
sub eof
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->eof');
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => $self->{oStorageCRead}->eof() ? true : false, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');
|
||||
|
||||
$self->{oStorageCRead}->close();
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => true, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Get a filter result
|
||||
####################################################################################################################################
|
||||
sub result
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strClass,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->result', \@_,
|
||||
{name => 'strClass'},
|
||||
);
|
||||
|
||||
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->result($strClass));
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'xResult', value => $xResult, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Get all filter results
|
||||
####################################################################################################################################
|
||||
sub resultAll
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');
|
||||
|
||||
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->resultAll());
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'xResultAll', value => $xResult, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
1;
|
||||
163
lib/pgBackRest/Storage/StorageWrite.pm
Normal file
163
lib/pgBackRest/Storage/StorageWrite.pm
Normal file
@ -0,0 +1,163 @@
|
||||
####################################################################################################################################
|
||||
# C Storage Write Interface
|
||||
####################################################################################################################################
|
||||
package pgBackRest::Storage::StorageWrite;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
use Fcntl qw(:mode);
|
||||
use File::stat qw{lstat};
|
||||
use JSON::PP;
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
sub new
|
||||
{
|
||||
my $class = shift;
|
||||
|
||||
# Create the class hash
|
||||
my $self = {};
|
||||
bless $self, $class;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
(
|
||||
my $strOperation,
|
||||
$self->{oStorage},
|
||||
$self->{oStorageCWrite},
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'oStorage'},
|
||||
{name => 'oStorageCWrite'},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'self', value => $self}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Open the file
|
||||
####################################################################################################################################
|
||||
sub open
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');
|
||||
|
||||
$self->{oStorageCWrite}->open();
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => true, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Write data
|
||||
####################################################################################################################################
|
||||
sub write
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my (
|
||||
$strOperation,
|
||||
$rtBuffer,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->write', \@_,
|
||||
{name => 'rtBuffer'},
|
||||
);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'iActualSize', value => $self->{oStorageCWrite}->write($$rtBuffer)}
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Close the file
|
||||
####################################################################################################################################
|
||||
sub close
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');
|
||||
|
||||
$self->{oStorageCWrite}->close();
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'bResult', value => true, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Get a filter result
|
||||
####################################################################################################################################
|
||||
sub result
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strClass,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->result', \@_,
|
||||
{name => 'strClass'},
|
||||
);
|
||||
|
||||
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->result($strClass));
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'xResult', value => $xResult, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Get all filter results
|
||||
####################################################################################################################################
|
||||
sub resultAll
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');
|
||||
|
||||
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->resultAll());
|
||||
|
||||
return logDebugReturn
|
||||
(
|
||||
$strOperation,
|
||||
{name => 'xResultAll', value => $xResult, trace => true},
|
||||
);
|
||||
}
|
||||
|
||||
1;
|
||||
12
libc/LibC.h
12
libc/LibC.h
@ -181,3 +181,15 @@ Free memory context in destructors
|
||||
***********************************************************************************************************************************/
|
||||
#define MEM_CONTEXT_XS_DESTROY(memContext) \
|
||||
memContextFree(memContext)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Create new string from an SV
|
||||
***********************************************************************************************************************************/
|
||||
#define STR_NEW_SV(param) \
|
||||
(SvOK(param) ? strNewN(SvPV_nolen(param), SvCUR(param)) : NULL)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Create const buffer from an SV
|
||||
***********************************************************************************************************************************/
|
||||
#define BUF_CONST_SV(param) \
|
||||
(SvOK(param) ? BUF(SvPV_nolen(param), SvCUR(param)) : NULL)
|
||||
|
||||
@ -50,6 +50,7 @@ These includes are from the src directory. There is no Perl-specific code in th
|
||||
***********************************************************************************************************************************/
|
||||
#include "common/crypto/common.h"
|
||||
#include "common/error.h"
|
||||
#include "common/io/io.h"
|
||||
#include "common/lock.h"
|
||||
#include "config/config.h"
|
||||
#include "config/define.h"
|
||||
@ -69,9 +70,11 @@ XSH includes
|
||||
|
||||
These includes define data structures that are required for the C to Perl interface but are not part of the regular C source.
|
||||
***********************************************************************************************************************************/
|
||||
#include "xs/crypto/cipherBlock.xsh"
|
||||
#include "xs/crypto/hash.xsh"
|
||||
#include "xs/common/encode.xsh"
|
||||
#include "xs/storage/storage.xsh"
|
||||
#include "xs/storage/storageRead.xsh"
|
||||
#include "xs/storage/storageWrite.xsh"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Module definition
|
||||
@ -97,8 +100,9 @@ INCLUDE: xs/common/lock.xs
|
||||
INCLUDE: xs/config/config.xs
|
||||
INCLUDE: xs/config/configTest.xs
|
||||
INCLUDE: xs/config/define.xs
|
||||
INCLUDE: xs/crypto/cipherBlock.xs
|
||||
INCLUDE: xs/crypto/hash.xs
|
||||
INCLUDE: xs/crypto/random.xs
|
||||
INCLUDE: xs/postgres/pageChecksum.xs
|
||||
INCLUDE: xs/storage/storage.xs
|
||||
INCLUDE: xs/storage/storageRead.xs
|
||||
INCLUDE: xs/storage/storageWrite.xs
|
||||
|
||||
@ -42,6 +42,7 @@ my @stryCFile =
|
||||
(
|
||||
'LibC.c',
|
||||
|
||||
'command/backup/pageChecksum.c',
|
||||
'command/command.c',
|
||||
'common/compress/gzip/common.c',
|
||||
'common/compress/gzip/compress.c',
|
||||
@ -61,8 +62,14 @@ my @stryCFile =
|
||||
'common/io/filter/group.c',
|
||||
'common/io/filter/size.c',
|
||||
'common/io/handleWrite.c',
|
||||
'common/io/http/cache.c',
|
||||
'common/io/http/client.c',
|
||||
'common/io/http/common.c',
|
||||
'common/io/http/header.c',
|
||||
'common/io/http/query.c',
|
||||
'common/io/io.c',
|
||||
'common/io/read.c',
|
||||
'common/io/tls/client.c',
|
||||
'common/io/write.c',
|
||||
'common/lock.c',
|
||||
'common/log.c',
|
||||
@ -86,10 +93,19 @@ my @stryCFile =
|
||||
'config/load.c',
|
||||
'config/parse.c',
|
||||
'perl/config.c',
|
||||
'protocol/client.c',
|
||||
'protocol/command.c',
|
||||
'protocol/helper.c',
|
||||
'protocol/parallel.c',
|
||||
'protocol/parallelJob.c',
|
||||
'protocol/server.c',
|
||||
'postgres/pageChecksum.c',
|
||||
'storage/posix/read.c',
|
||||
'storage/posix/storage.c',
|
||||
'storage/posix/write.c',
|
||||
'storage/s3/read.c',
|
||||
'storage/s3/storage.c',
|
||||
'storage/s3/write.c',
|
||||
'storage/helper.c',
|
||||
'storage/read.c',
|
||||
'storage/storage.c',
|
||||
|
||||
@ -19,8 +19,6 @@ use lib dirname($0) . '/../lib';
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Storage::Local;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
use pgBackRest::Version;
|
||||
|
||||
use pgBackRestBuild::Build;
|
||||
@ -28,6 +26,9 @@ use pgBackRestBuild::Build::Common;
|
||||
use pgBackRestBuild::Config::Data;
|
||||
use pgBackRestBuild::Error::Data;
|
||||
|
||||
use pgBackRestTest::Common::Storage;
|
||||
use pgBackRestTest::Common::StoragePosix;
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl function and constant exports
|
||||
####################################################################################################################################
|
||||
@ -74,8 +75,6 @@ my $rhExport =
|
||||
{
|
||||
&BLD_EXPORTTYPE_SUB => [qw(
|
||||
pageChecksum
|
||||
pageChecksumBufferTest
|
||||
pageChecksumTest
|
||||
)],
|
||||
},
|
||||
|
||||
@ -149,7 +148,7 @@ my $rhExport =
|
||||
'storage' =>
|
||||
{
|
||||
&BLD_EXPORTTYPE_SUB => [qw(
|
||||
storagePosixPathRemove
|
||||
storageRepoFree
|
||||
)],
|
||||
},
|
||||
|
||||
@ -172,8 +171,8 @@ sub buildXsAll
|
||||
my @stryBuilt;
|
||||
|
||||
# Storage
|
||||
my $oStorage = new pgBackRest::Storage::Local(
|
||||
$strBuildPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false}));
|
||||
my $oStorage = new pgBackRestTest::Common::Storage(
|
||||
$strBuildPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
|
||||
|
||||
# Build interface file
|
||||
my $strContent =
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
pgBackRest::LibC::Cipher::Block T_PTROBJ
|
||||
pgBackRest::LibC::Crypto::Hash T_PTROBJ
|
||||
pgBackRest::LibC::Storage T_PTROBJ
|
||||
pgBackRest::LibC::StorageRead T_PTROBJ
|
||||
pgBackRest::LibC::StorageWrite T_PTROBJ
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Block Cipher Perl Exports
|
||||
#
|
||||
# XS wrapper for functions in cipher/block.c.
|
||||
####################################################################################################################################
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Cipher::Block
|
||||
|
||||
####################################################################################################################################
|
||||
pgBackRest::LibC::Cipher::Block
|
||||
new(class, mode, type, key, keySize, digest = NULL)
|
||||
const char *class
|
||||
U32 mode
|
||||
const char *type
|
||||
unsigned char *key
|
||||
I32 keySize
|
||||
const char *digest
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
CHECK(type != NULL);
|
||||
CHECK(key != NULL);
|
||||
CHECK(keySize != 0);
|
||||
|
||||
// Not much point to this but it keeps the var from being unused
|
||||
if (strcmp(class, PACKAGE_NAME_LIBC "::Cipher::Block") != 0)
|
||||
croak("unexpected class name '%s'", class);
|
||||
|
||||
MEM_CONTEXT_XS_NEW_BEGIN("cipherBlockXs")
|
||||
{
|
||||
RETVAL = memNew(sizeof(CipherBlockXs));
|
||||
RETVAL->memContext = MEM_CONTEXT_XS();
|
||||
|
||||
RETVAL->pxPayload = cipherBlockNew(mode, cipherType(STR(type)), BUF(key, keySize), digest == NULL ? NULL : STR(digest));
|
||||
}
|
||||
MEM_CONTEXT_XS_NEW_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
process(self, source)
|
||||
pgBackRest::LibC::Cipher::Block self
|
||||
SV *source
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
MEM_CONTEXT_XS_BEGIN(self->memContext)
|
||||
{
|
||||
STRLEN tSize;
|
||||
const unsigned char *sourcePtr = (const unsigned char *)SvPV(source, tSize);
|
||||
|
||||
RETVAL = NEWSV(0, ioBufferSize());
|
||||
SvPOK_only(RETVAL);
|
||||
|
||||
if (tSize > 0)
|
||||
{
|
||||
size_t outBufferUsed = 0;
|
||||
|
||||
do
|
||||
{
|
||||
SvGROW(RETVAL, outBufferUsed + ioBufferSize());
|
||||
Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize());
|
||||
|
||||
ioFilterProcessInOut(self->pxPayload, BUF(sourcePtr, tSize), outBuffer);
|
||||
outBufferUsed += bufUsed(outBuffer);
|
||||
}
|
||||
while (ioFilterInputSame(self->pxPayload));
|
||||
|
||||
SvCUR_set(RETVAL, outBufferUsed);
|
||||
}
|
||||
else
|
||||
SvCUR_set(RETVAL, 0);
|
||||
}
|
||||
MEM_CONTEXT_XS_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
flush(self)
|
||||
pgBackRest::LibC::Cipher::Block self
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
MEM_CONTEXT_XS_BEGIN(self->memContext)
|
||||
{
|
||||
RETVAL = NEWSV(0, ioBufferSize());
|
||||
SvPOK_only(RETVAL);
|
||||
|
||||
size_t outBufferUsed = 0;
|
||||
|
||||
do
|
||||
{
|
||||
SvGROW(RETVAL, outBufferUsed + ioBufferSize());
|
||||
Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize());
|
||||
|
||||
ioFilterProcessInOut(self->pxPayload, NULL, outBuffer);
|
||||
outBufferUsed += bufUsed(outBuffer);
|
||||
}
|
||||
while (!ioFilterDone(self->pxPayload));
|
||||
|
||||
SvCUR_set(RETVAL, outBufferUsed);
|
||||
}
|
||||
MEM_CONTEXT_XS_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
DESTROY(self)
|
||||
pgBackRest::LibC::Cipher::Block self
|
||||
CODE:
|
||||
MEM_CONTEXT_XS_DESTROY(self->memContext);
|
||||
@ -1,17 +0,0 @@
|
||||
/***********************************************************************************************************************************
|
||||
Block Cipher XS Header
|
||||
***********************************************************************************************************************************/
|
||||
#include "common/assert.h"
|
||||
#include "common/crypto/cipherBlock.h"
|
||||
#include "common/io/io.h"
|
||||
#include "common/memContext.h"
|
||||
|
||||
// Encrypt/decrypt modes
|
||||
#define CIPHER_MODE_ENCRYPT ((int)cipherModeEncrypt)
|
||||
#define CIPHER_MODE_DECRYPT ((int)cipherModeDecrypt)
|
||||
|
||||
typedef struct CipherBlockXs
|
||||
{
|
||||
MemContext *memContext;
|
||||
IoFilter *pxPayload;
|
||||
} CipherBlockXs, *pgBackRest__LibC__Cipher__Block;
|
||||
@ -4,72 +4,6 @@
|
||||
# XS wrapper for functions in cipher/hash.c.
|
||||
####################################################################################################################################
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Crypto::Hash
|
||||
|
||||
####################################################################################################################################
|
||||
pgBackRest::LibC::Crypto::Hash
|
||||
new(class, type)
|
||||
const char *class
|
||||
const char *type
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
// Don't warn when class param is used
|
||||
(void)class;
|
||||
|
||||
MEM_CONTEXT_XS_NEW_BEGIN("cryptoHashXs")
|
||||
{
|
||||
RETVAL = memNew(sizeof(CryptoHashXs));
|
||||
RETVAL->memContext = MEM_CONTEXT_XS();
|
||||
RETVAL->pxPayload = cryptoHashNew(strNew(type));
|
||||
}
|
||||
MEM_CONTEXT_XS_NEW_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
process(self, message)
|
||||
pgBackRest::LibC::Crypto::Hash self
|
||||
SV *message
|
||||
CODE:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
STRLEN messageSize;
|
||||
const void *messagePtr = SvPV(message, messageSize);
|
||||
|
||||
if (messageSize > 0)
|
||||
ioFilterProcessIn(self->pxPayload, BUF(messagePtr, messageSize));
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
result(self)
|
||||
pgBackRest::LibC::Crypto::Hash self
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
const String *hash = varStr(ioFilterResult(self->pxPayload));
|
||||
|
||||
RETVAL = newSV(strSize(hash));
|
||||
SvPOK_only(RETVAL);
|
||||
strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash));
|
||||
SvCUR_set(RETVAL, strSize(hash));
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
DESTROY(self)
|
||||
pgBackRest::LibC::Crypto::Hash self
|
||||
CODE:
|
||||
MEM_CONTEXT_XS_DESTROY(self->memContext);
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
|
||||
|
||||
####################################################################################################################################
|
||||
|
||||
@ -2,11 +2,3 @@
|
||||
Cryptographic Hashes XS Header
|
||||
***********************************************************************************************************************************/
|
||||
#include "common/crypto/hash.h"
|
||||
#include "common/io/filter/filter.intern.h"
|
||||
#include "common/memContext.h"
|
||||
|
||||
typedef struct CryptoHashXs
|
||||
{
|
||||
MemContext *memContext;
|
||||
IoFilter *pxPayload;
|
||||
} CryptoHashXs, *pgBackRest__LibC__Crypto__Hash;
|
||||
|
||||
@ -20,42 +20,3 @@ CODE:
|
||||
ERROR_XS_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
bool
|
||||
pageChecksumTest(page, blockNo, pageSize, ignoreWalId, ignoreWalOffset)
|
||||
const char *page
|
||||
U32 blockNo
|
||||
U32 pageSize
|
||||
U32 ignoreWalId
|
||||
U32 ignoreWalOffset
|
||||
CODE:
|
||||
RETVAL = false;
|
||||
|
||||
ERROR_XS_BEGIN()
|
||||
{
|
||||
RETVAL = pageChecksumTest(
|
||||
(const unsigned char *)page, blockNo, pageSize, ignoreWalId, ignoreWalOffset);
|
||||
}
|
||||
ERROR_XS_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
bool
|
||||
pageChecksumBufferTest(pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset)
|
||||
const char *pageBuffer
|
||||
U32 pageBufferSize
|
||||
U32 blockNoBegin
|
||||
U32 pageSize
|
||||
U32 ignoreWalId
|
||||
U32 ignoreWalOffset
|
||||
CODE:
|
||||
RETVAL = false;
|
||||
|
||||
ERROR_XS_BEGIN()
|
||||
{
|
||||
RETVAL = pageChecksumBufferTest(
|
||||
(const unsigned char *)pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset);
|
||||
}
|
||||
ERROR_XS_END();
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
|
||||
@ -2,19 +2,409 @@
|
||||
# Storage Exports
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Storage
|
||||
|
||||
####################################################################################################################################
|
||||
pgBackRest::LibC::Storage
|
||||
new(class, type, path)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
const String *class = STR_NEW_SV($arg);
|
||||
const String *type = STR_NEW_SV($arg);
|
||||
const String *path = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::Storage"));
|
||||
|
||||
if (strEqZ(type, "<LOCAL>"))
|
||||
{
|
||||
memContextSwitch(MEM_CONTEXT_XS_OLD());
|
||||
RETVAL = storagePosixNew(
|
||||
path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
|
||||
storagePathEnforceSet((Storage *)RETVAL, false);
|
||||
memContextSwitch(MEM_CONTEXT_XS_TEMP());
|
||||
}
|
||||
else if (strEqZ(type, "<REPO>"))
|
||||
{
|
||||
CHECK(path == NULL);
|
||||
RETVAL = (Storage *)storageRepoWrite();
|
||||
}
|
||||
else if (strEqZ(type, "<DB>"))
|
||||
{
|
||||
CHECK(path == NULL);
|
||||
|
||||
memContextSwitch(MEM_CONTEXT_XS_OLD());
|
||||
RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
|
||||
storagePathEnforceSet((Storage *)RETVAL, false);
|
||||
memContextSwitch(MEM_CONTEXT_XS_TEMP());
|
||||
}
|
||||
else
|
||||
THROW_FMT(AssertError, "unexpected storage type '%s'", strPtr(type));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
copy(self, source, destination)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead source
|
||||
pgBackRest::LibC::StorageWrite destination
|
||||
CODE:
|
||||
RETVAL = storageCopyNP(source, destination);
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
exists(self, fileExp)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *fileExp = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
RETVAL = storageExistsNP(self, fileExp);
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
get(self, read)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead read
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
Buffer *buffer = storageGetNP(read);
|
||||
|
||||
if (buffer != NULL)
|
||||
{
|
||||
if (bufUsed(buffer) == 0)
|
||||
RETVAL = newSVpv("", 0);
|
||||
else
|
||||
RETVAL = newSVpv((char *)bufPtr(buffer), bufUsed(buffer));
|
||||
}
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
info(self, pathExp, ignoreMissing)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
bool ignoreMissing
|
||||
CODE:
|
||||
RETVAL = NULL;
|
||||
|
||||
StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = ignoreMissing);
|
||||
|
||||
if (info.exists)
|
||||
{
|
||||
String *json = storageManifestXsInfo(NULL, &info);
|
||||
RETVAL = newSVpv((char *)strPtr(json), strSize(json));
|
||||
}
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
list(self, pathExp, ignoreMissing, sortAsc, expression)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
bool ignoreMissing
|
||||
bool sortAsc
|
||||
const String *expression = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
StringList *fileList = strLstSort(
|
||||
storageListP(self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false,
|
||||
.expression = expression), sortAsc ? sortOrderAsc : sortOrderDesc);
|
||||
|
||||
const String *fileListJson = jsonFromVar(varNewVarLst(varLstNewStrLst(fileList)), 0);
|
||||
|
||||
RETVAL = newSVpv(strPtr(fileListJson), strSize(fileListJson));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
manifest(self, pathExp, filter=NULL)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
const String *filter = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
StorageManifestXsCallbackData data = {.storage = self, .json = strNew("{"), .pathRoot = pathExp, .filter = filter};
|
||||
|
||||
// If a path is specified
|
||||
StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = true);
|
||||
|
||||
if (!info.exists || info.type == storageTypePath)
|
||||
{
|
||||
storageInfoListP(
|
||||
self, data.pathRoot, storageManifestXsCallback, &data,
|
||||
.errorOnMissing = storageFeature(self, storageFeaturePath) ? true : false);
|
||||
}
|
||||
// Else a file is specified
|
||||
else
|
||||
{
|
||||
info.name = strBase(storagePath(self, pathExp));
|
||||
strCat(data.json, strPtr(storageManifestXsInfo(NULL, &info)));
|
||||
}
|
||||
|
||||
strCat(data.json, "}");
|
||||
|
||||
RETVAL = newSVpv((char *)strPtr(data.json), strSize(data.json));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
pathCreate(self, pathExp, mode, ignoreExists, createParent)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
const String *mode = STR_NEW_SV($arg);
|
||||
bool ignoreExists
|
||||
bool createParent
|
||||
CODE:
|
||||
if (storageFeature(self, storageFeaturePath))
|
||||
storagePathCreateP(
|
||||
self, pathExp, .mode = mode ? cvtZToIntBase(strPtr(mode), 8) : 0, .errorOnExists = !ignoreExists,
|
||||
.noParentCreate = !createParent);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
pathExists(self, pathExp)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
RETVAL = true;
|
||||
|
||||
if (storageFeature(self, storageFeaturePath))
|
||||
RETVAL = storagePathExistsNP(self, pathExp);
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
pathGet(self, pathExp)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
String *path = storagePathNP(self, pathExp);
|
||||
RETVAL = newSVpv((char *)strPtr(path), strSize(path));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
pathRemove(self, pathExp, ignoreMissing, recurse)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
bool ignoreMissing
|
||||
bool recurse
|
||||
CODE:
|
||||
storagePathRemoveP(
|
||||
self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, .recurse = recurse);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
pathSync(self, pathExp)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *pathExp = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
storagePathSyncNP(self, pathExp);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
UV
|
||||
put(self, write, buffer)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite write
|
||||
const Buffer *buffer = BUF_CONST_SV($arg);
|
||||
CODE:
|
||||
storagePutNP(write, buffer);
|
||||
RETVAL = buffer ? bufUsed(buffer) : 0;
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
readDrain(self, read)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead read
|
||||
CODE:
|
||||
RETVAL = false;
|
||||
|
||||
// Read and discard all IO (this is useful for processing filters)
|
||||
if (ioReadOpen(storageReadIo(read)))
|
||||
{
|
||||
Buffer *buffer = bufNew(ioBufferSize());
|
||||
|
||||
do
|
||||
{
|
||||
ioRead(storageReadIo(read), buffer);
|
||||
bufUsedZero(buffer);
|
||||
}
|
||||
while (!ioReadEof(storageReadIo(read)));
|
||||
|
||||
ioReadClose(storageReadIo(read));
|
||||
RETVAL = true;
|
||||
}
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
remove(self, fileExp, ignoreMissing)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
const String *fileExp = STR_NEW_SV($arg);
|
||||
bool ignoreMissing
|
||||
CODE:
|
||||
storageRemoveP(self, fileExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
cipherType(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
CODE:
|
||||
if (cfgOptionStr(cfgOptRepoCipherType) == NULL || cipherType(cfgOptionStr(cfgOptRepoCipherType)) == cipherTypeNone)
|
||||
RETVAL = NULL;
|
||||
else
|
||||
RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherType));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
cipherPass(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
CODE:
|
||||
RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherPass));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
type(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::Storage self
|
||||
CODE:
|
||||
RETVAL = strPtr(storageType(self));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
storagePosixPathRemove(path, errorOnMissing, recurse)
|
||||
const char *path
|
||||
bool errorOnMissing
|
||||
bool recurse
|
||||
storageRepoFree()
|
||||
CODE:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
storagePathRemoveP(
|
||||
storagePosixNew(strNew("/"), 0640, 750, true, NULL), strNew(path), .errorOnMissing = errorOnMissing,
|
||||
.recurse = recurse);
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
storageHelperFree();
|
||||
|
||||
222
libc/xs/storage/storage.xsh
Normal file
222
libc/xs/storage/storage.xsh
Normal file
@ -0,0 +1,222 @@
|
||||
/***********************************************************************************************************************************
|
||||
Storage XS Header
|
||||
***********************************************************************************************************************************/
|
||||
#include "command/backup/pageChecksum.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/compress/gzip/compress.h"
|
||||
#include "common/compress/gzip/decompress.h"
|
||||
#include "common/crypto/cipherBlock.h"
|
||||
#include "common/io/filter/size.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/type/convert.h"
|
||||
#include "common/type/json.h"
|
||||
#include "postgres/interface.h"
|
||||
#include "storage/helper.h"
|
||||
#include "storage/storage.intern.h"
|
||||
|
||||
typedef Storage *pgBackRest__LibC__Storage;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Manifest callback
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageManifestXsCallbackData
|
||||
{
|
||||
const Storage *storage;
|
||||
const String *pathRoot;
|
||||
const String *path;
|
||||
String *json;
|
||||
const String *filter;
|
||||
} StorageManifestXsCallbackData;
|
||||
|
||||
String *
|
||||
storageManifestXsInfo(const String *path, const StorageInfo *info)
|
||||
{
|
||||
String *json = strNew("");
|
||||
|
||||
if (info->name != NULL)
|
||||
{
|
||||
strCatFmt(
|
||||
json, "%s:", strPtr(jsonFromStr(path == NULL ? info->name : strNewFmt("%s/%s", strPtr(path), strPtr(info->name)))));
|
||||
}
|
||||
|
||||
strCatFmt(json, "{\"group\":%s,\"user\":%s,\"type\":\"", strPtr(jsonFromStr(info->group)), strPtr(jsonFromStr(info->user)));
|
||||
|
||||
switch (info->type)
|
||||
{
|
||||
case storageTypeFile:
|
||||
{
|
||||
strCatFmt(
|
||||
json, "f\",\"mode\":\"%04o\",\"modification_time\":%" PRId64 ",\"size\":%" PRIu64 "}", info->mode,
|
||||
(int64_t)info->timeModified, info->size);
|
||||
break;
|
||||
}
|
||||
|
||||
case storageTypeLink:
|
||||
{
|
||||
strCatFmt(json, "l\",\"link_destination\":%s}", strPtr(jsonFromStr(info->linkDestination)));
|
||||
break;
|
||||
}
|
||||
|
||||
case storageTypePath:
|
||||
{
|
||||
strCatFmt(json, "d\",\"mode\":\"%04o\"}", info->mode);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
void
|
||||
storageManifestXsCallback(void *callbackData, const StorageInfo *info)
|
||||
{
|
||||
StorageManifestXsCallbackData *data = (StorageManifestXsCallbackData *)callbackData;
|
||||
|
||||
if (data->path == NULL || !strEqZ(info->name, "."))
|
||||
{
|
||||
if (!strEqZ(info->name, ".") && data->filter && !strEq(data->filter, info->name))
|
||||
return;
|
||||
|
||||
if (strSize(data->json) != 1)
|
||||
strCat(data->json, ",");
|
||||
|
||||
strCat(data->json, strPtr(storageManifestXsInfo(data->path, info)));
|
||||
|
||||
if (info->type == storageTypePath)
|
||||
{
|
||||
if (!strEqZ(info->name, "."))
|
||||
{
|
||||
StorageManifestXsCallbackData dataSub =
|
||||
{
|
||||
.storage = data->storage,
|
||||
.json = data->json,
|
||||
.pathRoot = data->pathRoot,
|
||||
.path = data->path == NULL ? info->name : strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)),
|
||||
};
|
||||
|
||||
storageInfoListNP(
|
||||
dataSub.storage, strNewFmt("%s/%s", strPtr(dataSub.pathRoot), strPtr(dataSub.path)), storageManifestXsCallback,
|
||||
&dataSub);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Add IO filter
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const String *paramJson)
|
||||
{
|
||||
VariantList *paramList = paramJson ? jsonToVarLst(paramJson) : NULL;
|
||||
|
||||
if (strEqZ(filter, "pgBackRest::Storage::Filter::CipherBlock"))
|
||||
{
|
||||
ioFilterGroupAdd(
|
||||
filterGroup,
|
||||
cipherBlockNew(
|
||||
varUInt64Force(varLstGet(paramList, 0)) ? cipherModeEncrypt : cipherModeDecrypt,
|
||||
cipherType(varStr(varLstGet(paramList, 1))), BUFSTR(varStr(varLstGet(paramList, 2))), NULL));
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Sha"))
|
||||
{
|
||||
ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR));
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Common::Io::Handle"))
|
||||
{
|
||||
ioFilterGroupAdd(filterGroup, ioSizeNew());
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum"))
|
||||
{
|
||||
ioFilterGroupAdd(
|
||||
filterGroup,
|
||||
pageChecksumNew(
|
||||
varUInt64Force(varLstGet(paramList, 0)), PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT,
|
||||
(uint64_t)varUIntForce(varLstGet(paramList, 1)) << 32 | varUIntForce(varLstGet(paramList, 2))));
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gzip"))
|
||||
{
|
||||
if (strEqZ(varStr(varLstGet(paramList, 0)), "compress"))
|
||||
{
|
||||
ioFilterGroupAdd(
|
||||
filterGroup, gzipCompressNew(varUIntForce(varLstGet(paramList, 2)), varBoolForce(varLstGet(paramList, 1))));
|
||||
}
|
||||
else
|
||||
{
|
||||
ioFilterGroupAdd(filterGroup, gzipDecompressNew(varBoolForce(varLstGet(paramList, 1))));
|
||||
}
|
||||
}
|
||||
else
|
||||
THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get result from IO filter
|
||||
***********************************************************************************************************************************/
|
||||
String *
|
||||
storageFilterXsResult(const IoFilterGroup *filterGroup, const String *filter)
|
||||
{
|
||||
const Variant *result;
|
||||
|
||||
if (strEqZ(filter, "pgBackRest::Storage::Filter::Sha"))
|
||||
{
|
||||
result = ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR);
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Common::Io::Handle"))
|
||||
{
|
||||
result = ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR);
|
||||
}
|
||||
else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum"))
|
||||
{
|
||||
result = ioFilterGroupResult(filterGroup, PAGE_CHECKSUM_FILTER_TYPE_STR);
|
||||
}
|
||||
else
|
||||
THROW_FMT(AssertError, "unable to get result for invalid filter '%s'", strPtr(filter));
|
||||
|
||||
if (result == NULL)
|
||||
THROW_FMT(AssertError, "unable to find result for filter '%s'", strPtr(filter));
|
||||
|
||||
return jsonFromVar(result, 0);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get results from all IO filters
|
||||
***********************************************************************************************************************************/
|
||||
String *
|
||||
storageFilterXsResultAll(const IoFilterGroup *filterGroup)
|
||||
{
|
||||
const VariantList *filterList = kvKeyList(varKv(ioFilterGroupResultAll(filterGroup)));
|
||||
String *result = strNew("{");
|
||||
|
||||
for (unsigned int filterIdx = 0; filterIdx < varLstSize(filterList); filterIdx++)
|
||||
{
|
||||
const String *filter = varStr(varLstGet(filterList, filterIdx));
|
||||
const String *filterPerl = NULL;
|
||||
|
||||
if (strEq(filter, CRYPTO_HASH_FILTER_TYPE_STR))
|
||||
{
|
||||
filterPerl = STRDEF("pgBackRest::Storage::Filter::Sha");
|
||||
}
|
||||
else if (strEq(filter, SIZE_FILTER_TYPE_STR))
|
||||
{
|
||||
filterPerl = STRDEF("pgBackRest::Common::Io::Handle");
|
||||
}
|
||||
else if (strEq(filter, PAGE_CHECKSUM_FILTER_TYPE_STR))
|
||||
{
|
||||
filterPerl = STRDEF("pgBackRest::Backup::Filter::PageChecksum");
|
||||
}
|
||||
|
||||
if (filterPerl != NULL)
|
||||
{
|
||||
if (strSize(result) > 1)
|
||||
strCat(result, ",");
|
||||
|
||||
strCatFmt(
|
||||
result, "%s:%s", strPtr(jsonFromStr(filterPerl)), strPtr(storageFilterXsResult(filterGroup, filterPerl)));
|
||||
}
|
||||
}
|
||||
|
||||
strCat(result, "}");
|
||||
|
||||
return result;
|
||||
}
|
||||
159
libc/xs/storage/storageRead.xs
Normal file
159
libc/xs/storage/storageRead.xs
Normal file
@ -0,0 +1,159 @@
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
# Storage Read Exports
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageRead
|
||||
|
||||
####################################################################################################################################
|
||||
pgBackRest::LibC::StorageRead
|
||||
new(class, storage, file, ignoreMissing)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
const String *class = STR_NEW_SV($arg);
|
||||
pgBackRest::LibC::Storage storage
|
||||
const String *file = STR_NEW_SV($arg);
|
||||
bool ignoreMissing
|
||||
CODE:
|
||||
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageRead"));
|
||||
|
||||
RETVAL = storageReadMove(storageNewReadP(storage, file, .ignoreMissing = ignoreMissing), MEM_CONTEXT_XS_OLD());
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
filterAdd(self, filter, param)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
const String *filter = STR_NEW_SV($arg);
|
||||
const String *param = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(self));
|
||||
storageFilterXsAdd(filterGroup, filter, param);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
open(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
CODE:
|
||||
RETVAL = ioReadOpen(storageReadIo(self));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
SV *
|
||||
read(self, bufferSize)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
U32 bufferSize
|
||||
CODE:
|
||||
RETVAL = NEWSV(0, bufferSize);
|
||||
SvPOK_only(RETVAL);
|
||||
|
||||
Buffer *bufferRead = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL), bufferSize);
|
||||
ioRead(storageReadIo(self), bufferRead);
|
||||
|
||||
SvCUR_set(RETVAL, bufUsed(bufferRead));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
bool
|
||||
eof(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
CODE:
|
||||
RETVAL = ioReadEof(storageReadIo(self));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
close(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
CODE:
|
||||
ioReadClose(storageReadIo(self));
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
result(self, filter)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
const String *filter = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
RETVAL = strPtr(storageFilterXsResult(ioReadFilterGroup(storageReadIo(self)), filter));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
resultAll(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
CODE:
|
||||
RETVAL = strPtr(storageFilterXsResultAll(ioReadFilterGroup(storageReadIo(self))));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
DESTROY(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageRead self
|
||||
CODE:
|
||||
storageReadFree(self);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
4
libc/xs/storage/storageRead.xsh
Normal file
4
libc/xs/storage/storageRead.xsh
Normal file
@ -0,0 +1,4 @@
|
||||
/***********************************************************************************************************************************
|
||||
Storage Read XS Header
|
||||
***********************************************************************************************************************************/
|
||||
typedef StorageRead *pgBackRest__LibC__StorageRead;
|
||||
146
libc/xs/storage/storageWrite.xs
Normal file
146
libc/xs/storage/storageWrite.xs
Normal file
@ -0,0 +1,146 @@
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
# Storage Write Exports
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageWrite
|
||||
|
||||
####################################################################################################################################
|
||||
pgBackRest::LibC::StorageWrite
|
||||
new(class, storage, file, mode, user, group, timeModified, atomic, pathCreate)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
const String *class = STR_NEW_SV($arg);
|
||||
pgBackRest::LibC::Storage storage
|
||||
const String *file = STR_NEW_SV($arg);
|
||||
U32 mode
|
||||
const String *user = STR_NEW_SV($arg);
|
||||
const String *group = STR_NEW_SV($arg);
|
||||
IV timeModified
|
||||
bool atomic
|
||||
bool pathCreate
|
||||
CODE:
|
||||
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageWrite"));
|
||||
|
||||
RETVAL = storageWriteMove(
|
||||
storageNewWriteP(
|
||||
storage, file, .modeFile = mode, .user = user, .group = group, .timeModified = (time_t)timeModified,
|
||||
.noCreatePath = storageFeature(storage, storageFeaturePath) ? !pathCreate : false, .noSyncPath = !atomic,
|
||||
.noAtomic = !atomic),
|
||||
MEM_CONTEXT_XS_OLD());
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
filterAdd(self, filter, param)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
const String *filter = STR_NEW_SV($arg);
|
||||
const String *param = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(self));
|
||||
storageFilterXsAdd(filterGroup, filter, param);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
open(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
CODE:
|
||||
ioWriteOpen(storageWriteIo(self));
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
UV
|
||||
write(self, buffer)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
const Buffer *buffer = BUF_CONST_SV($arg);
|
||||
CODE:
|
||||
ioWrite(storageWriteIo(self), buffer);
|
||||
RETVAL = bufUsed(buffer);
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
close(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
CODE:
|
||||
ioWriteClose(storageWriteIo(self));
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
result(self, filter)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
const String *filter = STR_NEW_SV($arg);
|
||||
CODE:
|
||||
RETVAL = strPtr(storageFilterXsResult(ioWriteFilterGroup(storageWriteIo(self)), filter));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
const char *
|
||||
resultAll(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
CODE:
|
||||
RETVAL = strPtr(storageFilterXsResultAll(ioWriteFilterGroup(storageWriteIo(self))));
|
||||
OUTPUT:
|
||||
RETVAL
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
|
||||
####################################################################################################################################
|
||||
void
|
||||
DESTROY(self)
|
||||
PREINIT:
|
||||
MEM_CONTEXT_XS_TEMP_BEGIN()
|
||||
{
|
||||
INPUT:
|
||||
pgBackRest::LibC::StorageWrite self
|
||||
CODE:
|
||||
storageWriteFree(self);
|
||||
CLEANUP:
|
||||
}
|
||||
MEM_CONTEXT_XS_TEMP_END();
|
||||
4
libc/xs/storage/storageWrite.xsh
Normal file
4
libc/xs/storage/storageWrite.xsh
Normal file
@ -0,0 +1,4 @@
|
||||
/***********************************************************************************************************************************
|
||||
Storage Write XS Header
|
||||
***********************************************************************************************************************************/
|
||||
typedef StorageWrite *pgBackRest__LibC__StorageWrite;
|
||||
@ -52,6 +52,7 @@ SRCS = \
|
||||
command/archive/push/protocol.c \
|
||||
command/archive/push/push.c \
|
||||
command/backup/common.c \
|
||||
command/backup/pageChecksum.c \
|
||||
command/expire/expire.c \
|
||||
command/help/help.c \
|
||||
command/info/info.c \
|
||||
@ -433,7 +434,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.
|
||||
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o
|
||||
|
||||
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/pageChecksum.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/cipherBlock.xsh ../libc/xs/crypto/hash.xsh
|
||||
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o
|
||||
|
||||
postgres/interface.o: postgres/interface.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/version.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
|
||||
|
||||
@ -90,7 +90,8 @@ restoreFile(
|
||||
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
|
||||
|
||||
Buffer *buffer = bufNew(ioBufferSize());
|
||||
ioReadOpen(read);
|
||||
|
||||
CHECK(ioReadOpen(read));
|
||||
|
||||
do
|
||||
{
|
||||
|
||||
@ -31,7 +31,7 @@ struct ErrorType
|
||||
/***********************************************************************************************************************************
|
||||
Maximum allowed number of nested try blocks
|
||||
***********************************************************************************************************************************/
|
||||
#define ERROR_TRY_MAX 32
|
||||
#define ERROR_TRY_MAX 64
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
States for each try
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
1676
src/perl/libc.auto.c
1676
src/perl/libc.auto.c
File diff suppressed because it is too large
Load Diff
@ -28,7 +28,7 @@ struct Storage
|
||||
mode_t modeFile;
|
||||
mode_t modePath;
|
||||
bool write;
|
||||
bool pathResolve;
|
||||
bool pathEnforce;
|
||||
StoragePathExpressionCallback pathExpressionFunction;
|
||||
};
|
||||
|
||||
@ -73,6 +73,7 @@ storageNew(
|
||||
this->path = strDup(path);
|
||||
this->modeFile = modeFile;
|
||||
this->modePath = modePath;
|
||||
this->pathEnforce = true;
|
||||
this->write = write;
|
||||
this->pathExpressionFunction = pathExpressionFunction;
|
||||
|
||||
@ -493,8 +494,8 @@ storagePath(const Storage *this, const String *pathExp)
|
||||
// Make sure the base storage path is contained within the path expression
|
||||
if (this->path != NULL && !strEqZ(this->path, "/"))
|
||||
{
|
||||
if (!strBeginsWith(pathExp, this->path) ||
|
||||
!(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/'))
|
||||
if (this->pathEnforce && (!strBeginsWith(pathExp, this->path) ||
|
||||
!(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/')))
|
||||
{
|
||||
THROW_FMT(AssertError, "absolute path '%s' is not in base path '%s'", strPtr(pathExp), strPtr(this->path));
|
||||
}
|
||||
@ -586,10 +587,6 @@ storagePathCreate(const Storage *this, const String *pathExp, StoragePathCreateP
|
||||
ASSERT(this->interface.pathCreate != NULL && storageFeature(this, storageFeaturePath));
|
||||
ASSERT(this->write);
|
||||
|
||||
// It doesn't make sense to combine these parameters because if we are creating missing parent paths why error when they exist?
|
||||
// If this somehow wasn't caught in testing, the worst case is that the path would not be created and an error would be thrown.
|
||||
ASSERT(!(param.noParentCreate && param.errorOnExists));
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Build the path
|
||||
@ -781,6 +778,22 @@ storageInterface(const Storage *this)
|
||||
FUNCTION_TEST_RETURN(this->interface);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Set whether absolute paths are required to be in the base path
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storagePathEnforceSet(Storage *this, bool enforce)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE, this);
|
||||
FUNCTION_TEST_PARAM(BOOL, enforce);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
this->pathEnforce = enforce;
|
||||
|
||||
FUNCTION_TEST_RETURN_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get the storage type (posix, cifs, etc.)
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -82,11 +82,15 @@ Storage *storageNew(
|
||||
StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters
|
||||
Getters/Setters
|
||||
***********************************************************************************************************************************/
|
||||
void *storageDriver(const Storage *this);
|
||||
StorageInterface storageInterface(const Storage *this);
|
||||
|
||||
// The option is intended to be used only with the Perl interface since Perl is not tidy about where it reads. It should be
|
||||
// removed when the Perl interface is removed.
|
||||
void storagePathEnforceSet(Storage *this, bool enforce);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Macros for function logging
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -276,13 +276,6 @@ unit:
|
||||
- name: encode-perl
|
||||
total: 1
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: http-client-perl
|
||||
total: 2
|
||||
|
||||
coverage:
|
||||
Common/Http/Client: partial
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: reg-exp
|
||||
total: 3
|
||||
@ -423,82 +416,18 @@ unit:
|
||||
|
||||
test:
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: filter-cipher-block-perl
|
||||
total: 2
|
||||
- name: perl
|
||||
total: 13
|
||||
|
||||
coverage:
|
||||
Storage/Filter/CipherBlock: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: filter-gzip-perl
|
||||
total: 3
|
||||
|
||||
coverage:
|
||||
Storage/Filter/Gzip: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: filter-sha-perl
|
||||
total: 2
|
||||
|
||||
coverage:
|
||||
Storage/Filter/Sha: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: posix-perl
|
||||
total: 10
|
||||
|
||||
coverage:
|
||||
Storage/Posix/Driver: partial
|
||||
Storage/Posix/FileRead: partial
|
||||
Storage/Posix/FileWrite: partial
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: s3-auth-perl
|
||||
total: 5
|
||||
|
||||
coverage:
|
||||
Storage/S3/Auth: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: s3-cert-perl
|
||||
total: 1
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: s3-request-perl
|
||||
total: 2
|
||||
|
||||
coverage:
|
||||
Storage/S3/Request: partial
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: s3-perl
|
||||
total: 7
|
||||
|
||||
coverage:
|
||||
Storage/S3/Driver: partial
|
||||
Storage/S3/FileRead: partial
|
||||
Storage/S3/FileWrite: full
|
||||
|
||||
vm:
|
||||
- co7
|
||||
- u14
|
||||
- u16
|
||||
- u18
|
||||
- d8
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: local-perl
|
||||
total: 10
|
||||
|
||||
coverage:
|
||||
Storage/Local: partial
|
||||
Storage/Storage: partial
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: helper-perl
|
||||
total: 4
|
||||
total: 3
|
||||
|
||||
coverage:
|
||||
Storage/Helper: partial
|
||||
Storage/Helper: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: cifs
|
||||
@ -840,7 +769,3 @@ performance:
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: archive
|
||||
total: 1
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: io
|
||||
total: 1
|
||||
|
||||
@ -994,8 +994,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
|
||||
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
|
||||
P00 ERROR: [055]: unable to stat '[TEST_PATH]/db-master/db/base_tbs': No such file or directory
|
||||
P00 INFO: backup command end: aborted with exception [055]
|
||||
P00 ERROR: [073]: unable to list file info for missing path '[TEST_PATH]/db-master/db/base_tbs'
|
||||
P00 INFO: backup command end: aborted with exception [073]
|
||||
|
||||
incr backup - invalid tablespace in $PGDATA (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --stanza=db backup
|
||||
|
||||
@ -257,7 +257,7 @@ full backup - resume (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 WARN: backup [BACKUP-FULL-1] missing in repository removed from backup.info
|
||||
P00 WARN: backup [BACKUP-FULL-1] missing manifest removed from backup.info
|
||||
P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it looks like the postmaster is running and the backup will probably not be consistent
|
||||
P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed
|
||||
P00 TEST: PgBaCkReStTeSt-BACKUP-RESUME-PgBaCkReStTeSt
|
||||
@ -743,7 +743,7 @@ incr backup - resume and add tablespace 2 (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 WARN: backup [BACKUP-INCR-1] missing in repository removed from backup.info
|
||||
P00 WARN: backup [BACKUP-INCR-1] missing manifest removed from backup.info
|
||||
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
|
||||
P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum
|
||||
P00 WARN: aborted backup [BACKUP-INCR-2] of same type exists, will be cleaned to remove invalid files and resumed
|
||||
@ -3350,6 +3350,7 @@ P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
|
||||
P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0
|
||||
P00 INFO: diff backup size = 9B
|
||||
P00 INFO: new backup label = [BACKUP-DIFF-6]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin
|
||||
P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired
|
||||
@ -3558,6 +3559,7 @@ P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
|
||||
P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0
|
||||
P00 INFO: diff backup size = 9B
|
||||
P00 INFO: new backup label = [BACKUP-DIFF-7]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin
|
||||
P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired
|
||||
|
||||
@ -29,6 +29,8 @@ stanza-create db - stanza create (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online --force stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-2] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
|
||||
@ -5,6 +5,8 @@ stanza-create db - create required data for stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
|
||||
@ -209,7 +209,7 @@ stanza-create db - gunzip fail on forced stanza-create (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --no-online --force stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
|
||||
P00 ERROR: [041]: unable to open '[TEST_PATH]/db-master/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz': Permission denied
|
||||
P00 ERROR: [041]: unable to open file '[TEST_PATH]/db-master/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz' for read: [13] Permission denied
|
||||
P00 INFO: stanza-create command end: aborted with exception [041]
|
||||
|
||||
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
|
||||
|
||||
@ -16,12 +16,16 @@ P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL s
|
||||
HINT: is archive_command configured in postgresql.conf?
|
||||
HINT: has a stanza-create been performed?
|
||||
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-upgrade command end: aborted with exception [055]
|
||||
|
||||
stanza-create db - successfully create the stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -67,6 +71,8 @@ stanza-create db - do not fail on rerun of stanza-create - info files exist and
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 INFO: stanza-create was already performed
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -113,6 +119,8 @@ stanza-create db - fail on database mismatch without force option (backup host)
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 ERROR: [028]: backup info file or archive info file invalid
|
||||
HINT: use stanza-upgrade if the database has been upgraded or use --force
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: aborted with exception [028]
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -158,6 +166,8 @@ stanza-upgrade db - already up to date (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 INFO: the stanza data is already up to date
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -209,6 +219,8 @@ stanza-create db - force create archive.info from gz file (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 ERROR: [055]: archive information missing and repo is encrypted and info file(s) are missing, --force cannot be used
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: aborted with exception [055]
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -246,6 +258,8 @@ stanza-upgrade db - successful upgrade creates additional history (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -309,6 +323,8 @@ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_
|
||||
P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
|
||||
P00 INFO: full backup size = 48MB
|
||||
P00 INFO: new backup label = [BACKUP-FULL-1]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin
|
||||
P00 INFO: remove archive path: /archive/db/9.3-1
|
||||
@ -381,6 +397,8 @@ stanza-create db - fail no force to recreate the stanza from backups (backup hos
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 ERROR: [055]: backup information missing and repo is encrypted and info file(s) are missing, --force cannot be used
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: aborted with exception [055]
|
||||
|
||||
+ supplemental file: /archive/db/archive.info
|
||||
@ -407,6 +425,8 @@ stanza-create db - use force to recreate the stanza from backups (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 ERROR: [055]: backup information missing and repo is encrypted and info file(s) are missing, --force cannot be used
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-create command end: aborted with exception [055]
|
||||
|
||||
+ supplemental file: /archive/db/archive.info
|
||||
@ -432,6 +452,8 @@ stanza-upgrade db - successfully upgrade - no info file mismatch (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -495,6 +517,8 @@ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_
|
||||
P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
|
||||
P00 INFO: full backup size = 48MB
|
||||
P00 INFO: new backup label = [BACKUP-FULL-2]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
@ -695,6 +719,8 @@ stanza-delete db - successfully delete the stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-delete
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-delete command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: stanza-delete command end: completed successfully
|
||||
|
||||
db must not exist for successful delete
|
||||
|
||||
@ -136,7 +136,7 @@ sub testDefLoad
|
||||
|
||||
# Set module type variables
|
||||
$hTestDefHash->{$strModule}{$strTest}{&TESTDEF_C} =
|
||||
$strModuleType eq TESTDEF_UNIT && $strTest !~ /\-perl$/ ? true : false;
|
||||
$strModuleType eq TESTDEF_UNIT && $strTest !~ /perl$/ ? true : false;
|
||||
$hTestDefHash->{$strModule}{$strTest}{&TESTDEF_INTEGRATION} = $strModuleType eq TESTDEF_INTEGRATION ? true : false;
|
||||
$hTestDefHash->{$strModule}{$strTest}{&TESTDEF_EXPECT} = $bExpect;
|
||||
$hTestDefHash->{$strModule}{$strTest}{&TESTDEF_CONTAINER} = $bContainer;
|
||||
|
||||
@ -27,8 +27,7 @@ use pgBackRest::Common::String;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Storage::Local;
|
||||
use pgBackRest::Storage::S3::Driver;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::HostGroupTest;
|
||||
@ -168,7 +167,7 @@ sub forceStorageMode
|
||||
);
|
||||
|
||||
# Mode commands are ignored on S3
|
||||
if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER)
|
||||
if ($oStorage->type() ne STORAGE_S3)
|
||||
{
|
||||
executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
|
||||
}
|
||||
@ -203,7 +202,7 @@ sub forceStorageMove
|
||||
);
|
||||
|
||||
# If S3 then use storage commands to remove
|
||||
if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER)
|
||||
if ($oStorage->type() eq STORAGE_S3)
|
||||
{
|
||||
hostGroupGet()->hostGet(HOST_S3)->executeS3(
|
||||
'mv' . ($bRecurse ? ' --recursive' : '') . ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) .
|
||||
@ -244,8 +243,8 @@ sub forceStorageOwner
|
||||
{name => 'bRecurse', optional => true, default => false},
|
||||
);
|
||||
|
||||
# Mode commands are ignored on S3
|
||||
if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER)
|
||||
# Owner commands are ignored on S3
|
||||
if ($oStorage->type() ne STORAGE_S3)
|
||||
{
|
||||
executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
|
||||
}
|
||||
@ -278,11 +277,19 @@ sub forceStorageRemove
|
||||
);
|
||||
|
||||
# If S3 then use storage commands to remove
|
||||
if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER)
|
||||
if ($oStorage->type() eq STORAGE_S3)
|
||||
{
|
||||
$oStorage->remove($strPathExp, {bRecurse => $bRecurse});
|
||||
my $oInfo = $oStorage->info($strPathExp, {bIgnoreMissing => true});
|
||||
|
||||
if (defined($oInfo) && $oInfo->{type} eq 'f')
|
||||
{
|
||||
$oStorage->remove($strPathExp);
|
||||
}
|
||||
else
|
||||
{
|
||||
$oStorage->pathRemove($strPathExp, {bRecurse => true});
|
||||
}
|
||||
}
|
||||
# Else remove using filesystem commands
|
||||
else
|
||||
{
|
||||
executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
|
||||
|
||||
@ -19,10 +19,11 @@ use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
use pgBackRest::Storage::Local;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Storage;
|
||||
use pgBackRest::Version;
|
||||
|
||||
use pgBackRestTest::Common::BuildTest;
|
||||
use pgBackRestTest::Common::DefineTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::LogTest;
|
||||
@ -155,7 +156,7 @@ sub process
|
||||
$self->{bFirstTest} = true;
|
||||
|
||||
# Initialize test storage
|
||||
$oStorage = new pgBackRest::Storage::Local($self->testPath(), new pgBackRest::Storage::Posix::Driver());
|
||||
$oStorage = new pgBackRest::Storage::Storage(STORAGE_LOCAL, {strPath => $self->testPath()});
|
||||
|
||||
# Generate backrest exe
|
||||
$self->{strBackRestExe} = testRunExe(
|
||||
|
||||
@ -15,6 +15,7 @@ use Exporter qw(import);
|
||||
our @EXPORT = qw();
|
||||
use Fcntl ':mode';
|
||||
use File::Basename qw(dirname);
|
||||
use File::stat qw{lstat};
|
||||
use Storable qw(dclone);
|
||||
|
||||
use pgBackRest::Archive::Info;
|
||||
@ -27,9 +28,8 @@ use pgBackRest::Config::Config;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
use pgBackRest::Storage::S3::Driver;
|
||||
use pgBackRest::Version;
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
use pgBackRestTest::Env::Host::HostBaseTest;
|
||||
use pgBackRestTest::Env::Host::HostS3Test;
|
||||
@ -262,65 +262,68 @@ sub backupEnd
|
||||
}
|
||||
|
||||
# Make sure tablespace links are correct
|
||||
if (($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) && $self->hasLink())
|
||||
if ($self->hasLink())
|
||||
{
|
||||
my $hTablespaceManifest = storageRepo()->manifest(
|
||||
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
|
||||
|
||||
# Remove . and ..
|
||||
delete($hTablespaceManifest->{'.'});
|
||||
delete($hTablespaceManifest->{'..'});
|
||||
|
||||
# Iterate file links
|
||||
for my $strFile (sort(keys(%{$hTablespaceManifest})))
|
||||
if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink())
|
||||
{
|
||||
# Make sure the link is in the expected manifest
|
||||
my $hManifestTarget =
|
||||
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
|
||||
my $hTablespaceManifest = storageRepo()->manifest(
|
||||
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
|
||||
|
||||
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
|
||||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
|
||||
# Remove . and ..
|
||||
delete($hTablespaceManifest->{'.'});
|
||||
delete($hTablespaceManifest->{'..'});
|
||||
|
||||
# Iterate file links
|
||||
for my $strFile (sort(keys(%{$hTablespaceManifest})))
|
||||
{
|
||||
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
|
||||
# Make sure the link is in the expected manifest
|
||||
my $hManifestTarget =
|
||||
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
|
||||
|
||||
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
|
||||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
|
||||
{
|
||||
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
|
||||
}
|
||||
|
||||
# Make sure the link really is a link
|
||||
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
|
||||
{
|
||||
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
|
||||
}
|
||||
|
||||
# Make sure the link destination is correct
|
||||
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
|
||||
|
||||
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
|
||||
{
|
||||
confess &log(ERROR,
|
||||
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
|
||||
"'$hTablespaceManifest->{$strFile}{link_destination}'");
|
||||
}
|
||||
}
|
||||
|
||||
# Make sure the link really is a link
|
||||
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
|
||||
# Iterate manifest targets
|
||||
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
|
||||
{
|
||||
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
|
||||
}
|
||||
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
|
||||
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
|
||||
|
||||
# Make sure the link destination is correct
|
||||
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
|
||||
|
||||
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
|
||||
{
|
||||
confess &log(ERROR,
|
||||
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
|
||||
"'$hTablespaceManifest->{$strFile}{link_destination}'");
|
||||
# Make sure the target exists as a link on disk
|
||||
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
|
||||
!defined($hTablespaceManifest->{$strTablespaceId}))
|
||||
{
|
||||
confess &log(ERROR,
|
||||
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Iterate manifest targets
|
||||
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
|
||||
# Else there should not be a tablespace directory at all
|
||||
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
|
||||
{
|
||||
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
|
||||
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
|
||||
|
||||
# Make sure the target exists as a link on disk
|
||||
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
|
||||
!defined($hTablespaceManifest->{$strTablespaceId}))
|
||||
{
|
||||
confess &log(ERROR,
|
||||
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
|
||||
}
|
||||
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
|
||||
}
|
||||
}
|
||||
# Else there should not be a tablespace directory at all
|
||||
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
|
||||
{
|
||||
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
|
||||
}
|
||||
|
||||
# Check that latest link exists unless repo links are disabled
|
||||
my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST);
|
||||
@ -495,7 +498,8 @@ sub backupCompare
|
||||
my $lRepoSize =
|
||||
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
|
||||
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
|
||||
(storageRepo()->info(STORAGE_REPO_BACKUP . "/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->size;
|
||||
(storageRepo()->info(STORAGE_REPO_BACKUP .
|
||||
"/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->{size};
|
||||
|
||||
if (defined($lRepoSize) &&
|
||||
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
|
||||
@ -1882,7 +1886,7 @@ sub restoreCompare
|
||||
|
||||
if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
|
||||
{
|
||||
my $oStat = storageTest()->info($oActualManifest->dbPathGet($strSectionPath, $strName));
|
||||
my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName));
|
||||
|
||||
# When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty
|
||||
# sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison
|
||||
@ -2049,8 +2053,8 @@ sub backupDestination {return shift->{strBackupDestination}}
|
||||
sub backrestExe {return testRunGet()->backrestExe()}
|
||||
sub bogusHost {return shift->{bBogusHost}}
|
||||
sub hardLink {return shift->{bHardLink}}
|
||||
sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER}
|
||||
sub isFS {storageRepo()->driver()->className() ne STORAGE_S3_DRIVER}
|
||||
sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)}
|
||||
sub isFS {storageRepo()->type() ne STORAGE_S3}
|
||||
sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
|
||||
sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
|
||||
sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}
|
||||
|
||||
@ -224,7 +224,7 @@ sub manifestFileCreate
|
||||
my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode);
|
||||
|
||||
# Stat the file
|
||||
my $oStat = storageTest()->info($strPathFile);
|
||||
my $oStat = lstat($strPathFile);
|
||||
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
|
||||
@ -338,7 +338,7 @@ sub manifestLinkCreate
|
||||
my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination);
|
||||
|
||||
# Stat the link
|
||||
my $oStat = storageTest()->info($strDbFile);
|
||||
my $oStat = lstat($strDbFile);
|
||||
|
||||
# Check for errors in stat
|
||||
if (!defined($oStat))
|
||||
@ -360,7 +360,7 @@ sub manifestLinkCreate
|
||||
(defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}";
|
||||
}
|
||||
|
||||
$oStat = storageTest()->info($strDestinationFile);
|
||||
$oStat = lstat($strDestinationFile);
|
||||
|
||||
my $strSection = MANIFEST_SECTION_TARGET_PATH;
|
||||
|
||||
@ -556,7 +556,7 @@ sub manifestTablespaceCreate
|
||||
# Load linked path into manifest
|
||||
my $strLinkPath = $self->tablespacePath($iOid);
|
||||
my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}";
|
||||
my $oStat = storageTest()->info($strLinkPath);
|
||||
my $oStat = lstat($strLinkPath);
|
||||
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
|
||||
@ -582,7 +582,7 @@ sub manifestTablespaceCreate
|
||||
}
|
||||
|
||||
# Load tablespace path into manifest
|
||||
$oStat = storageTest()->info($strTablespacePath);
|
||||
$oStat = lstat($strTablespacePath);
|
||||
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} =
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA};
|
||||
@ -599,7 +599,7 @@ sub manifestTablespaceCreate
|
||||
or confess "unable to link ${strLink} to ${strLinkPath}";
|
||||
|
||||
# Load link into the manifest
|
||||
$oStat = storageTest()->info($strLink);
|
||||
$oStat = lstat($strLink);
|
||||
my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}";
|
||||
|
||||
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
|
||||
|
||||
@ -163,10 +163,9 @@ sub run
|
||||
MANIFEST_SUBKEY_CHECKSUM, $strPgControlHash)}, true, "manifest updated for pg_control");
|
||||
|
||||
# Neither backup.manifest nor backup.manifest.copy written because size threshold not met
|
||||
$self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, ERROR_FILE_MISSING,
|
||||
"unable to open '$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT . "': No such file or directory");
|
||||
$self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST)}, ERROR_FILE_MISSING,
|
||||
"unable to open '$strBackupPath/" . FILE_MANIFEST . "': No such file or directory");
|
||||
$self->testResult(sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, "backup.manifest missing");
|
||||
$self->testResult(
|
||||
sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, false, "backup.manifest.copy missing");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# No prior checksum, no compression, no page checksum, no extra, no delta, no hasReference
|
||||
@ -180,8 +179,7 @@ sub run
|
||||
$lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true,
|
||||
'file copied to repo successfully');
|
||||
|
||||
$self->testException(sub {storageRepo()->openRead("$strFileRepo.gz")}, ERROR_FILE_MISSING,
|
||||
"unable to open '$strFileRepo.gz': No such file or directory");
|
||||
$self->testResult(sub {storageRepo()->exists("${strFileRepo}.gz")}, false, "${strFileRepo}.gz missing");
|
||||
|
||||
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
|
||||
$oBackupManifest,
|
||||
@ -212,18 +210,13 @@ sub run
|
||||
# Backup.manifest not written but backup.manifest.copy written because size threshold met
|
||||
$self->testResult(sub {storageTest()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, true,
|
||||
'backup.manifest.copy exists in repo');
|
||||
$self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST)}, ERROR_FILE_MISSING,
|
||||
"unable to open '$strBackupPath/" . FILE_MANIFEST . "': No such file or directory");
|
||||
$self->testResult(
|
||||
sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, 'backup.manifest.copy missing in repo');
|
||||
|
||||
storageTest()->remove($strFileRepo);
|
||||
storageTest()->remove($strPgControlRepo);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# No prior checksum, yes compression, yes page checksum, no extra, no delta, no hasReference
|
||||
$self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true,
|
||||
$strBackupLabel, true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef)}, ERROR_ASSERT,
|
||||
"iWalId is required in Backup::Filter::PageChecksum->new");
|
||||
|
||||
# Build the lsn start parameter to pass to the extra function
|
||||
my $hStartLsnParam =
|
||||
{
|
||||
@ -231,7 +224,6 @@ sub run
|
||||
iWalOffset => 0xFFFF,
|
||||
};
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# No prior checksum, yes compression, yes page checksum, yes extra, no delta, no hasReference
|
||||
($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) =
|
||||
backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true, $strBackupLabel, true,
|
||||
@ -243,8 +235,7 @@ sub run
|
||||
$lResultRepoSize == $lRepoFileCompressSize && $rResultExtra->{bValid}), true, 'file copied to repo successfully');
|
||||
|
||||
# Only the compressed version of the file exists
|
||||
$self->testException(sub {storageRepo()->openRead("$strFileRepo")}, ERROR_FILE_MISSING,
|
||||
"unable to open '$strFileRepo': No such file or directory");
|
||||
$self->testResult(sub {storageRepo()->exists("$strFileRepo")}, false, "only compressed version exists");
|
||||
|
||||
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
|
||||
$oBackupManifest,
|
||||
@ -368,7 +359,7 @@ sub run
|
||||
# do not ignoreMissing
|
||||
$self->testException(sub {backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, $strFileHash,
|
||||
false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, false, undef, true, false, undef)},
|
||||
ERROR_FILE_MISSING, "unable to open '$strFileDb.1': No such file or directory");
|
||||
ERROR_FILE_MISSING, "unable to open missing file '${strFileDb}.1' for read");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Restore the compressed file
|
||||
@ -518,7 +509,7 @@ sub run
|
||||
|
||||
$self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash,
|
||||
false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef)},
|
||||
ERROR_FILE_MISSING, "unable to open '$strFileRepo': No such file or directory");
|
||||
ERROR_FILE_MISSING, "unable to open missing file '${strFileRepo}' for read");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
|
||||
@ -1,173 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Request Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Common::CommonHttpClientPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use IO::Socket::SSL;
|
||||
use POSIX qw(strftime);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Http::Client;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
|
||||
use pgBackRestTest::Common::ContainerTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# Port to use for testing
|
||||
####################################################################################################################################
|
||||
use constant HTTPS_TEST_PORT => 9443;
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerResponse
|
||||
####################################################################################################################################
|
||||
sub httpsServerResponse
|
||||
{
|
||||
my $self = shift;
|
||||
my $iResponseCode = shift;
|
||||
my $strContent = shift;
|
||||
|
||||
# Write header
|
||||
$self->{oConnection}->write("HTTP/1.1 ${iResponseCode} GenericMessage\r\n");
|
||||
$self->{oConnection}->write(HTTP_HEADER_CONTENT_LENGTH . ': ' . (defined($strContent) ? length($strContent) : 0) . "\r\n");
|
||||
|
||||
# Write new line before content (even if there isn't any)
|
||||
$self->{oConnection}->write("\r\n");
|
||||
|
||||
# Write content
|
||||
if (defined($strContent))
|
||||
{
|
||||
$self->{oConnection}->write($strContent);
|
||||
}
|
||||
|
||||
# This will block until the connection is closed by the client
|
||||
$self->{oConnection}->read();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerAccept
|
||||
####################################################################################################################################
|
||||
sub httpsServerAccept
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Wait for a connection
|
||||
$self->{oConnection} = $self->{oSocketServer}->accept()
|
||||
or confess "failed to accept or handshake $!, $SSL_ERROR";
|
||||
&log(INFO, " * socket server connected");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServer
|
||||
####################################################################################################################################
|
||||
sub httpsServer
|
||||
{
|
||||
my $self = shift;
|
||||
my $fnServer = shift;
|
||||
|
||||
# Fork off the server
|
||||
if (fork() == 0)
|
||||
{
|
||||
# Run server function
|
||||
$fnServer->();
|
||||
|
||||
exit 0;
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Start the https testing server
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Open the domain socket
|
||||
$self->{oSocketServer} = IO::Socket::SSL->new(
|
||||
LocalAddr => '127.0.0.1', LocalPort => HTTPS_TEST_PORT, Listen => 1, SSL_cert_file => CERT_FAKE_SERVER,
|
||||
SSL_key_file => CERT_FAKE_SERVER_KEY)
|
||||
or confess "unable to open https server for testing: $!";
|
||||
&log(INFO, " * socket server open");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Stop the https testing server
|
||||
####################################################################################################################################
|
||||
sub cleanModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Shutdown server
|
||||
$self->{oSocketServer}->close();
|
||||
&log(INFO, " * socket server closed");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Test variables
|
||||
my $strTestHost = '127.0.0.1';
|
||||
my $strTestData = 'TESTDATA';
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('content-length defined'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200, $strTestData);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oHttpClient = $self->testResult(
|
||||
sub {new pgBackRest::Common::Http::Client(
|
||||
$strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false})},
|
||||
'[object]', 'new http client');
|
||||
|
||||
$self->testResult(sub {${$oHttpClient->responseBody()}}, $strTestData, 'response body read');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('retry'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->{oConnection}->write("HTTP/1.1 404 Error\r\nBogus-Header\r\n\r\n");
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->{oConnection}->write("HTTP/1.1 404 Error\r\nBogus-Header\r\n\r\n");
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200, $strTestData);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Common::Http::Client(
|
||||
$strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false, iTryTotal => 1})},
|
||||
ERROR_PROTOCOL, "http header 'Bogus-Header' requires colon separator");
|
||||
|
||||
$self->testResult(
|
||||
sub {new pgBackRest::Common::Http::Client(
|
||||
$strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false, iTryTotal => 2})},
|
||||
'[object]', 'successful retries');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -297,42 +297,6 @@ sub run
|
||||
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CRYPTO,
|
||||
"unable to parse '$strTestFile'" .
|
||||
"\nHINT: Is or was the repo encrypted?");
|
||||
|
||||
# Encryption
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("rm -rf ${strTestFile}*");
|
||||
|
||||
my $strCipherPass = 'x';
|
||||
my $strCipherPassSub = 'y';
|
||||
|
||||
# Unencrypted storage but a passphrase passed
|
||||
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false,
|
||||
strCipherPass => $strCipherPass})}, ERROR_ASSERT,
|
||||
"a user passphrase and sub passphrase are both required when encrypting");
|
||||
|
||||
# Unencrypted storage but a sub passphrase passed
|
||||
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false,
|
||||
strCipherPassSub => $strCipherPassSub})}, ERROR_ASSERT,
|
||||
"a user passphrase and sub passphrase are both required when encrypting");
|
||||
|
||||
# Create Encrypted storage
|
||||
my $oStorage = new pgBackRest::Storage::Local($self->testPath(), new pgBackRest::Storage::Posix::Driver(),
|
||||
{strCipherType => CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC, strCipherPassUser => $strCipherPass});
|
||||
|
||||
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {oStorage => $oStorage})}, ERROR_CRYPTO,
|
||||
"passphrase is required when storage is encrypted");
|
||||
|
||||
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, oStorage => $oStorage,
|
||||
strCipherPass => $strCipherPass})}, ERROR_ASSERT,
|
||||
"a user passphrase and sub passphrase are both required when encrypting");
|
||||
|
||||
$oIni = $self->testResult(sub {
|
||||
new pgBackRest::Common::Ini(
|
||||
$strTestFile,
|
||||
{bLoad => false, oStorage => $oStorage, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub})},
|
||||
'[object]', 'create new ini with encryption passphrases');
|
||||
$self->testResult(sub {($oIni->cipherPassSub() eq $strCipherPassSub) &&
|
||||
($oIni->cipherPass() eq $strCipherPass)}, true, ' new ini has encryption passphrases');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
|
||||
@ -198,8 +198,8 @@ sub run
|
||||
|
||||
# Attempt to save without proper path
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(sub {$oManifest->save()}, ERROR_PATH_MISSING,
|
||||
"unable to open '" . $strBackupManifestFile . "': No such file or directory");
|
||||
$self->testException(sub {$oManifest->save()}, ERROR_FILE_MISSING,
|
||||
"unable to open file '${strBackupManifestFile}' for write in missing path");
|
||||
|
||||
# Create path and save
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -226,8 +226,8 @@ sub run
|
||||
|
||||
# Build error if offline = true and no tablespace path
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(sub {$oManifest->build(storageDb(), $self->{strDbPath}, undef, false, false)}, ERROR_FILE_MISSING,
|
||||
"unable to stat '" . $self->{strDbPath} . "/" . MANIFEST_TARGET_PGTBLSPC . "': No such file or directory");
|
||||
$self->testException(sub {$oManifest->build(storageDb(), $self->{strDbPath}, undef, false, false)}, ERROR_PATH_MISSING,
|
||||
"unable to list file info for missing path '" . $self->{strDbPath} . "/" . MANIFEST_TARGET_PGTBLSPC . "'");
|
||||
|
||||
# bOnline = true tests
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -236,7 +236,11 @@ sub run
|
||||
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)});
|
||||
|
||||
# Create expected manifest from base
|
||||
my $oStorageTemp = $oManifestBase->{oStorage};
|
||||
$oManifestBase->{oStorage} = undef;
|
||||
my $oManifestExpected = dclone($oManifestBase);
|
||||
$oManifestBase->{oStorage} = $oStorageTemp;
|
||||
$oManifestExpected->{oStorage} = $oStorageTemp;
|
||||
|
||||
# Add global/pg_control file and PG_VERSION file and create a directory with a different modes than default
|
||||
storageDb()->put(storageDb()->openWrite($self->{strDbPath} . '/' . DB_FILE_PGCONTROL,
|
||||
@ -523,7 +527,11 @@ sub run
|
||||
|
||||
# Unskip code path coverage
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oStorageTemp = $oManifestExpected->{oStorage};
|
||||
$oManifestExpected->{oStorage} = undef;
|
||||
my $oManifestExpectedUnskip = dclone($oManifestExpected);
|
||||
$oManifestExpected->{oStorage} = $oStorageTemp;
|
||||
$oManifestExpectedUnskip->{oStorage} = $oStorageTemp;
|
||||
|
||||
# Change DB version to 93
|
||||
$oManifest = new pgBackRest::Manifest(
|
||||
@ -954,7 +962,7 @@ sub run
|
||||
storageDb()->remove('postgresql.auto.conf');
|
||||
storageDb()->remove('hosts');
|
||||
storageDb()->remove('pg_log/logfile');
|
||||
storageDb()->remove('global/exclude', {bRecurse => true});
|
||||
storageDb()->pathRemove('global/exclude', {bRecurse => true});
|
||||
|
||||
# Reload the manifest with version < 9.0
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -1229,7 +1237,11 @@ sub run
|
||||
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)});
|
||||
|
||||
# Create expected manifest from base
|
||||
my $oStorageTemp = $oManifestBase->{oStorage};
|
||||
$oManifestBase->{oStorage} = undef;
|
||||
my $oManifestExpected = dclone($oManifestBase);
|
||||
$oManifestBase->{oStorage} = $oStorageTemp;
|
||||
$oManifestExpected->{oStorage} = $oStorageTemp;
|
||||
|
||||
# Future timestamp on file
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -1255,7 +1267,11 @@ sub run
|
||||
|
||||
# Future timestamp in last manifest
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oStorageTemp = $oManifestExpected->{oStorage};
|
||||
$oManifestExpected->{oStorage} = undef;
|
||||
my $oLastManifest = dclone($oManifestExpected);
|
||||
$oManifestExpected->{oStorage} = $oStorageTemp;
|
||||
$oLastManifest->{oStorage} = $oStorageTemp;
|
||||
|
||||
# Set a backup label
|
||||
$oLastManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, BOGUS);
|
||||
@ -1507,7 +1523,12 @@ sub run
|
||||
{
|
||||
my $oManifest = new pgBackRest::Manifest($strBackupManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94,
|
||||
iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)});
|
||||
|
||||
my $oStorageTemp = $oManifestBase->{oStorage};
|
||||
$oManifestBase->{oStorage} = undef;
|
||||
my $oManifestExpected = dclone($oManifestBase);
|
||||
$oManifestBase->{oStorage} = $oStorageTemp;
|
||||
$oManifestExpected->{oStorage} = $oStorageTemp;
|
||||
|
||||
# Add a bogus file - all traces to be removed after the manifest has been built to simulate an inital manifest and avoid
|
||||
# missing files error
|
||||
|
||||
@ -878,7 +878,7 @@ sub run
|
||||
|
||||
$oHostBackup->backup(
|
||||
$strType, '$PGDATA is a substring of valid tblspc excluding / (file missing err expected)',
|
||||
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_FILE_MISSING});
|
||||
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_PATH_MISSING});
|
||||
|
||||
testFileRemove("${strTblSpcPath}/99999");
|
||||
}
|
||||
|
||||
@ -73,14 +73,12 @@ sub run
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3,
|
||||
bRepoEncrypt => $bEncrypt});
|
||||
|
||||
my $oStorage = storageRepo();
|
||||
|
||||
# Create compression extension
|
||||
my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : '';
|
||||
|
||||
# Create the wal path
|
||||
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
$oStorage->pathCreate($strWalPath, {bCreateParent => true});
|
||||
storageTest()->pathCreate($strWalPath, {bCreateParent => true});
|
||||
|
||||
# Create the test path for pg_control and generate pg_control for stanza-create
|
||||
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
|
||||
@ -97,7 +95,7 @@ sub run
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoMunge(
|
||||
$oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
|
||||
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
|
||||
}
|
||||
@ -121,12 +119,12 @@ sub run
|
||||
# Fix the database version
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoRestore($oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
|
||||
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oStorage->list(
|
||||
sub {storageRepo()->list(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
|
||||
"000000010000000100000001-${strWalHash}${strCompressExt}",
|
||||
'segment 2-4 not pushed', {iWaitSeconds => 5});
|
||||
@ -135,7 +133,7 @@ sub run
|
||||
$oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 5);
|
||||
|
||||
$self->testResult(
|
||||
sub {$oStorage->list(
|
||||
sub {storageRepo()->list(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
|
||||
"(000000010000000100000001-${strWalHash}${strCompressExt}, " .
|
||||
"000000010000000100000005-${strWalHash}${strCompressExt})",
|
||||
|
||||
@ -25,7 +25,6 @@ use pgBackRest::InfoCommon;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
@ -198,7 +197,7 @@ sub run
|
||||
storageRepo()->copy(
|
||||
storageRepo()->openRead(
|
||||
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz",
|
||||
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}]}),
|
||||
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}]}),
|
||||
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}");
|
||||
|
||||
$oHostBackup->stanzaCreate('force create archive.info from uncompressed file',
|
||||
|
||||
@ -1,127 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# I/O Performance Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Performance::PerformanceIoTest;
|
||||
use parent 'pgBackRestTest::Env::ConfigEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Storable qw(dclone);
|
||||
use Time::HiRes qw(gettimeofday);
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# initModule
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Load reference page data
|
||||
my $tPageBin = ${storageTest()->get($self->dataPath() . '/filecopy.table.bin')};
|
||||
|
||||
# Create large test file
|
||||
$self->{iTableLargeSize} = 32;
|
||||
$self->{strTableLargeFile} = 'table-large.bin';
|
||||
|
||||
my $oFileWrite = storageTest()->openWrite($self->{strTableLargeFile});
|
||||
|
||||
for (my $iIndex = 0; $iIndex < $self->{iTableLargeSize}; $iIndex++)
|
||||
{
|
||||
$oFileWrite->write(\$tPageBin);
|
||||
}
|
||||
|
||||
$oFileWrite->close();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("copy"))
|
||||
{
|
||||
# Setup the remote for testing remote storage
|
||||
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
|
||||
$self->optionTestSet(CFGOPT_PG_PATH, $self->testPath());
|
||||
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath());
|
||||
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
|
||||
$self->optionTestSet(CFGOPT_REPO_HOST, 'localhost');
|
||||
$self->optionTestSet(CFGOPT_REPO_HOST_USER, $self->backrestUser());
|
||||
$self->configTestLoad(CFGCMD_RESTORE);
|
||||
|
||||
protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP, undef, {strBackRestBin => $self->backrestExe()});
|
||||
storageRepo();
|
||||
|
||||
# Setup file info
|
||||
my $strFile = $self->{strTableLargeFile};
|
||||
my $strFileCopy = "${strFile}.copy";
|
||||
|
||||
my $iRunTotal = 1;
|
||||
&log(INFO, "time is average of ${iRunTotal} run(s)");
|
||||
|
||||
foreach my $bGzip (false, true)
|
||||
{
|
||||
foreach my $bRemote (false, true)
|
||||
{
|
||||
my $rhyFilter;
|
||||
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_SHA});
|
||||
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => 6}]}) if ($bGzip);
|
||||
|
||||
my $lTimeTotal = 0;
|
||||
my $lTimeBegin;
|
||||
|
||||
for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++)
|
||||
{
|
||||
# Get the remote or local for writing
|
||||
my $oStorageWrite = $bRemote ? storageRepo() : storageTest();
|
||||
|
||||
# Start the timer
|
||||
$lTimeBegin = gettimeofday();
|
||||
|
||||
# Copy the file
|
||||
storageTest()->copy(
|
||||
storageTest()->openRead($strFile, {rhyFilter => $rhyFilter}),
|
||||
$oStorageWrite->openWrite($strFileCopy));
|
||||
|
||||
# Record time
|
||||
$lTimeTotal += gettimeofday() - $lTimeBegin;
|
||||
|
||||
# Remove file so it can be copied again
|
||||
executeTest("sudo rm " . $oStorageWrite->pathGet($strFileCopy));
|
||||
}
|
||||
|
||||
# Calculate out output metrics
|
||||
my $fExecutionTime = int($lTimeTotal * 1000 / $iRunTotal) / 1000;
|
||||
my $fGbPerHour = int((60 * 60) * 1000 / ((1024 / $self->{iTableLargeSize}) * $fExecutionTime)) / 1000;
|
||||
|
||||
&log(INFO, "sha1 1, gz ${bGzip}, rmt ${bRemote}: ${fExecutionTime}s, ${fGbPerHour} GB/hr");
|
||||
}
|
||||
}
|
||||
|
||||
# Destroy protocol object
|
||||
protocolDestroy();
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -36,6 +36,8 @@ use pgBackRestTest::Env::Host::HostBaseTest;
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
use pgBackRestTest::Env::Host::HostDbTest;
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::Storage;
|
||||
use pgBackRestTest::Common::StoragePosix;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
@ -270,7 +272,7 @@ sub run
|
||||
executeTest("sudo chmod 400 ${strDir}");
|
||||
|
||||
$strComment = 'confirm master manifest->build executed';
|
||||
$oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN});
|
||||
$oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN});
|
||||
executeTest("sudo rmdir ${strDir}");
|
||||
|
||||
# Providing a sufficient archive-timeout, verify that the check command runs successfully now with valid
|
||||
@ -503,10 +505,10 @@ sub run
|
||||
my $strComment = 'confirm standby manifest->build executed';
|
||||
|
||||
# If there is an invalid host, the final error returned from check will be the inability to resolve the name which is
|
||||
# a read error instead of an open error
|
||||
# an open error instead of a read error
|
||||
if (!$oHostDbStandby->bogusHost())
|
||||
{
|
||||
$oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN});
|
||||
$oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN});
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -746,13 +748,19 @@ sub run
|
||||
{
|
||||
my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath);
|
||||
|
||||
# Create a zeroed sparse file in the test directory that is the same size as the filenode.map
|
||||
# Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the
|
||||
# posix driver directly to do this because handles cannot be passed back from the C code.
|
||||
my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix());
|
||||
|
||||
my $strTestTable = $self->testPath() . "/testtable";
|
||||
my $oDestinationFileIo = storageTest()->openWrite($strTestTable);
|
||||
my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable);
|
||||
$oDestinationFileIo->open();
|
||||
|
||||
# Truncate to the original size which will create a sparse file.
|
||||
truncate($oDestinationFileIo->handle(), $lSize);
|
||||
if (!truncate($oDestinationFileIo->handle(), $lSize))
|
||||
{
|
||||
confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle();
|
||||
}
|
||||
$oDestinationFileIo->close();
|
||||
|
||||
# Confirm the test filenode.map and the database test1 filenode.map are zeroed
|
||||
|
||||
@ -280,7 +280,7 @@ sub run
|
||||
# Change the permissions on the archived file so reconstruction fails
|
||||
executeTest('sudo chmod 220 ' . $strArchivedFile);
|
||||
$self->testException(sub {(new pgBackRest::Stanza())->stanzaCreate()}, ERROR_FILE_OPEN,
|
||||
"unable to open '" . $strArchivedFile . "': Permission denied");
|
||||
"unable to open file '${strArchivedFile}' for read");
|
||||
executeTest('sudo chmod 644 ' . $strArchivedFile);
|
||||
|
||||
# Clear the cached repo settings and change repo settings to encrypted
|
||||
@ -312,7 +312,10 @@ sub run
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1");
|
||||
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1/0000000100000001");
|
||||
storageRepo()->put($strArchivedFile, $tUnencryptedArchiveContent);
|
||||
storageRepo()->put(
|
||||
storageRepo()->openWrite(
|
||||
$strArchivedFile, {strCipherPass => new pgBackRest::Archive::Info($self->{strArchivePath})->cipherPassSub()}),
|
||||
$tUnencryptedArchiveContent);
|
||||
storageRepo()->pathCreate($strBackupPath); # Empty backup path - no backup in progress
|
||||
|
||||
# Confirm encrypted and create the stanza with force
|
||||
@ -477,7 +480,7 @@ sub run
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest('sudo chmod 220 ' . $self->{strArchivePath});
|
||||
$self->testException(sub {$oStanza->infoObject(STORAGE_REPO_ARCHIVE, $self->{strArchivePath})}, ERROR_FILE_OPEN,
|
||||
"unable to open '" . $self->{strArchivePath} . "/archive.info': Permission denied");
|
||||
"unable to open file '" . $self->{strArchivePath} . "/archive.info' for read");
|
||||
executeTest('sudo chmod 640 ' . $self->{strArchivePath});
|
||||
|
||||
# Reset force option --------
|
||||
@ -492,8 +495,8 @@ sub run
|
||||
forceStorageRemove(storageRepo(), storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT));
|
||||
executeTest('sudo chmod 220 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
|
||||
$self->testException(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, ERROR_FILE_OPEN,
|
||||
"unable to open '" . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) .
|
||||
"': Permission denied");
|
||||
"unable to open file '" . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) .
|
||||
"' for read");
|
||||
executeTest('sudo chmod 640 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
|
||||
}
|
||||
|
||||
@ -762,8 +765,8 @@ sub run
|
||||
executeTest("sudo chown 7777 " . $self->{strArchivePath});
|
||||
|
||||
lockStop();
|
||||
$self->testException(sub {$oStanza->stanzaDelete()}, ERROR_FILE_OPEN,
|
||||
"unable to remove file '" . $self->{strArchivePath} . "/" . ARCHIVE_INFO_FILE . "': Permission denied");
|
||||
$self->testException(sub {$oStanza->stanzaDelete()}, ERROR_FILE_REMOVE,
|
||||
"unable to remove '" . $self->{strArchivePath} . "/" . ARCHIVE_INFO_FILE . "'");
|
||||
|
||||
# Remove the repo
|
||||
executeTest("sudo rm -rf " . $self->{strArchivePath});
|
||||
|
||||
@ -1,285 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Tests for Block Cipher
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageFilterCipherBlockPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Fcntl qw(O_RDONLY);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:random :crypto);
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::CipherBlock;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Test data
|
||||
my $strFile = $self->testPath() . qw{/} . 'file.txt';
|
||||
my $strFileEncrypt = $self->testPath() . qw{/} . 'file.enc.txt';
|
||||
my $strFileDecrypt = $self->testPath() . qw{/} . 'file.dcr.txt';
|
||||
my $strFileBin = $self->testPath() . qw{/} . 'file.bin';
|
||||
my $strFileBinEncrypt = $self->testPath() . qw{/} . 'file.enc.bin';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileLength = length($strFileContent);
|
||||
my $oDriver = new pgBackRest::Storage::Posix::Driver();
|
||||
my $tCipherPass = 'areallybadkey';
|
||||
my $strCipherType = 'aes-256-cbc';
|
||||
my $tContent;
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('new()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Create an unencrypted file
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
|
||||
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})},
|
||||
ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS);
|
||||
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFile), BOGUS, $tCipherPass)},
|
||||
ERROR_ASSERT, "invalid cipher name '" . BOGUS . "'");
|
||||
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openWrite($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})},
|
||||
ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS);
|
||||
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openWrite($strFile), BOGUS, $tCipherPass)},
|
||||
ERROR_ASSERT, "invalid cipher name '" . BOGUS . "'");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('read() and write()'))
|
||||
{
|
||||
my $tBuffer;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Create an plaintext file
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
|
||||
|
||||
# Instantiate the cipher object - default action encrypt
|
||||
my $oEncryptIo = $self->testResult(sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openRead($strFile),
|
||||
$strCipherType, $tCipherPass)}, '[object]', 'new encrypt file');
|
||||
|
||||
$self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (header)');
|
||||
$self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (data)');
|
||||
$self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes');
|
||||
|
||||
$self->testResult(sub {$tBuffer ne $strFileContent}, true, ' data read is encrypted');
|
||||
|
||||
$self->testResult(sub {$oEncryptIo->close()}, true, ' close');
|
||||
$self->testResult(sub {$oEncryptIo->close()}, false, ' close again');
|
||||
|
||||
# tBuffer is now encrypted - test write decrypts correctly
|
||||
my $oDecryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileDecrypt),
|
||||
$strCipherType, $tCipherPass, {strMode => STORAGE_DECRYPT})},
|
||||
'[object]', ' new decrypt file');
|
||||
|
||||
$self->testResult(sub {$oDecryptFileIo->write(\$tBuffer)}, 32, ' write decrypted');
|
||||
$self->testResult(sub {$oDecryptFileIo->close()}, true, ' close');
|
||||
|
||||
$self->testResult(sub {${$self->storageTest()->get($strFileDecrypt)}}, $strFileContent, ' data written is decrypted');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = $strFileContent;
|
||||
my $oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileEncrypt),
|
||||
$strCipherType, $tCipherPass)},
|
||||
'[object]', 'new write encrypt');
|
||||
|
||||
$tContent = '';
|
||||
$self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, 0, ' attempt empty buffer write');
|
||||
|
||||
undef($tContent);
|
||||
$self->testException(
|
||||
sub {$oEncryptFileIo->write(\$tContent)}, ERROR_FILE_WRITE,
|
||||
"unable to write to '${strFileEncrypt}': Use of uninitialized value");
|
||||
|
||||
# Encrypted length is not known so use tBuffer then test that tBuffer was encrypted
|
||||
my $iWritten = $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, length($tBuffer), ' write encrypted');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
|
||||
$tContent = $self->storageTest()->get($strFileDecrypt);
|
||||
$self->testResult(sub {defined($tContent) && $tContent ne $strFileContent}, true, ' data written is encrypted');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
undef($tBuffer);
|
||||
# Open encrypted file for decrypting
|
||||
$oEncryptFileIo =
|
||||
$self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'new read encrypted file, decrypt');
|
||||
|
||||
# Try to read more than the length of the data expected to be output from the decrypt and confirm the decrypted length is
|
||||
# the same as the original decrypted content.
|
||||
$self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, $iFileLength+4)}, $iFileLength, ' read all bytes');
|
||||
|
||||
# Just because length is the same does not mean content is so confirm
|
||||
$self->testResult($tBuffer, $strFileContent, ' data read is decrypted');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
undef($tContent);
|
||||
undef($tBuffer);
|
||||
my $strFileBinHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031';
|
||||
|
||||
executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFileBin}");
|
||||
$self->testResult(
|
||||
sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBin)})}, $strFileBinHash, 'bin test - check sha1');
|
||||
|
||||
$tContent = ${storageTest()->get($strFileBin)};
|
||||
|
||||
$oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openWrite($strFileBinEncrypt), $strCipherType, $tCipherPass)},
|
||||
'[object]', ' new write encrypt');
|
||||
|
||||
$self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, length($tContent), ' write encrypted');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
$self->testResult(
|
||||
sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBinEncrypt)}) ne $strFileBinHash}, true,
|
||||
' check sha1 different');
|
||||
|
||||
my $oEncryptBinFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileBinEncrypt), $strCipherType, $tCipherPass,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'new read encrypted bin file');
|
||||
|
||||
$self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read 16777216 bytes');
|
||||
$self->testResult(sub {cryptoHashOne('sha1', $tBuffer)}, $strFileBinHash, ' check sha1 same as original');
|
||||
$self->testResult(sub {$oEncryptBinFileIo->close()}, true, ' close');
|
||||
|
||||
# Try to read the file with the wrong passphrase
|
||||
undef($tBuffer);
|
||||
undef($oEncryptBinFileIo);
|
||||
|
||||
$oEncryptBinFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileBinEncrypt), $strCipherType, BOGUS,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'new read Encrypted bin file with wrong passphrase');
|
||||
|
||||
$self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read all bytes');
|
||||
$self->testResult(sub {cryptoHashOne('sha1', $tBuffer) ne $strFileBinHash}, true, ' check sha1 NOT same as original');
|
||||
|
||||
# Test file against openssl to make sure they are compatible
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
undef($tBuffer);
|
||||
|
||||
$self->storageTest()->put($strFile, $strFileContent);
|
||||
|
||||
executeTest(
|
||||
"openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}");
|
||||
|
||||
$oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'read file encrypted by openssl');
|
||||
|
||||
$self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 8, ' read 8 bytes');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
$self->testResult(sub {$tBuffer}, $strFileContent, ' check content same as original');
|
||||
|
||||
$self->storageTest()->remove($strFile);
|
||||
$self->storageTest()->remove($strFileEncrypt);
|
||||
|
||||
$oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)},
|
||||
'[object]', 'write file to be read by openssl');
|
||||
|
||||
$self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 8, ' write 8 bytes');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
|
||||
executeTest(
|
||||
"openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}");
|
||||
|
||||
$self->testResult(sub {${$self->storageTest()->get($strFile)}}, $strFileContent, ' check content same as original');
|
||||
|
||||
# Test empty file against openssl to make sure they are compatible
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = '';
|
||||
|
||||
$self->storageTest()->put($strFile);
|
||||
|
||||
executeTest(
|
||||
"openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}");
|
||||
|
||||
$oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'read empty file encrypted by openssl');
|
||||
|
||||
$self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 0, ' read 0 bytes');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
$self->testResult(sub {$tBuffer}, '', ' check content same as original');
|
||||
|
||||
$self->storageTest()->remove($strFile);
|
||||
$self->storageTest()->remove($strFileEncrypt);
|
||||
|
||||
$oEncryptFileIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)},
|
||||
'[object]', 'write file to be read by openssl');
|
||||
|
||||
$self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 0, ' write 0 bytes');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, ' close');
|
||||
|
||||
executeTest(
|
||||
"openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}");
|
||||
|
||||
$self->testResult(sub {${$self->storageTest()->get($strFile)}}, undef, ' check content same as original');
|
||||
|
||||
# Error on empty file decrypt - an empty file that has been encrypted will be 32 bytes
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
undef($tBuffer);
|
||||
$self->storageTest()->put($strFileEncrypt);
|
||||
|
||||
$oEncryptFileIo =
|
||||
$self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::CipherBlock(
|
||||
$oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass,
|
||||
{strMode => STORAGE_DECRYPT})},
|
||||
'[object]', 'new read empty attempt decrypt');
|
||||
|
||||
$self->testException(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, ERROR_CRYPTO, 'cipher header missing');
|
||||
$self->testResult(sub {$oEncryptFileIo->close()}, true, 'close');
|
||||
|
||||
# OpenSSL should error on the empty file
|
||||
executeTest(
|
||||
"openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}",
|
||||
{iExpectedExitStatus => 1});
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,222 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Tests for Storage::Filter::Gzip module
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageFilterGzipPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Compress::Raw::Zlib qw(Z_OK Z_BUF_ERROR Z_DATA_ERROR);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Test data
|
||||
my $strFile = $self->testPath() . qw{/} . 'file.txt';
|
||||
my $strFileGz = "${strFile}.gz";
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileLength = length($strFileContent);
|
||||
my $oDriver = new pgBackRest::Storage::Posix::Driver();
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('errorCheck()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz))}, '[object]', 'new write');
|
||||
|
||||
$oGzipIo->{bWrite} = true;
|
||||
$self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_WRITE, "unable to deflate '${strFileGz}'");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oGzipIo->errorCheck(Z_OK)}, Z_OK, 'Z_OK');
|
||||
$self->testResult(sub {$oGzipIo->errorCheck(Z_BUF_ERROR)}, Z_OK, 'Z_BUF_ERROR');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oGzipIo->{bWrite} = false;
|
||||
$oGzipIo->{strCompressType} = STORAGE_DECOMPRESS;
|
||||
$self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}'");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('write()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz), {lCompressBufferMax => 4})},
|
||||
'[object]', 'new write compress');
|
||||
|
||||
my $tBuffer = substr($strFileContent, 0, 2);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 2, 2);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 4, 2);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 6, 2);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
|
||||
$tBuffer = '';
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 0, ' write 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
|
||||
executeTest("gzip -d ${strFileGz}");
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("gzip ${strFile}");
|
||||
my $tFile = ${storageTest()->get($strFileGz)};
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip(
|
||||
$oDriver->openWrite($strFile), {strCompressType => STORAGE_DECOMPRESS})}, '[object]', 'new write decompress');
|
||||
|
||||
$tBuffer = substr($tFile, 0, 10);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 10, ' write bytes');
|
||||
$tBuffer = substr($tFile, 10);
|
||||
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, length($tFile) - 10, ' write bytes');
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
|
||||
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('read()'))
|
||||
{
|
||||
my $tBuffer;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz), {bWantGzip => false, iLevel => 3})},
|
||||
'[object]', 'new write compress');
|
||||
$self->testResult($oGzipIo->{iLevel}, 3, ' check level');
|
||||
$self->testResult(sub {$oGzipIo->write(\$strFileContent, $iFileLength)}, $iFileLength, ' write');
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip(
|
||||
$oDriver->openRead($strFileGz), {bWantGzip => false, strCompressType => STORAGE_DECOMPRESS})},
|
||||
'[object]', 'new read decompress');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4)}, 4, ' read 4 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
$self->testResult(sub {$oGzipIo->close()}, false, ' close again');
|
||||
$self->testResult($tBuffer, $strFileContent, ' check content');
|
||||
|
||||
storageTest()->remove($strFileGz);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = 'AA';
|
||||
storageTest()->put($strFile, $strFileContent);
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile))},
|
||||
'[object]', 'new read compress');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 10, ' read 10 bytes (request 2)');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 18, ' read 18 bytes (request 2)');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes (request 2)');
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
|
||||
$self->testResult(sub {storageTest()->put($strFileGz, substr($tBuffer, 2))}, 28, ' put content');
|
||||
executeTest("gzip -df ${strFileGz}");
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = undef;
|
||||
|
||||
executeTest('cat ' . $self->dataPath() . "/filecopy.archive2.bin | gzip -c > ${strFileGz}");
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip(
|
||||
$oDriver->openRead($strFileGz), {lCompressBufferMax => 4096, strCompressType => STORAGE_DECOMPRESS})},
|
||||
'[object]', 'new read decompress');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
$self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content');
|
||||
|
||||
storageTest()->remove($strFileGz);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = undef;
|
||||
|
||||
executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFile}");
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {strCompressType => STORAGE_COMPRESS})},
|
||||
'[object]', 'new read compress');
|
||||
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes');
|
||||
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
|
||||
|
||||
$self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content');
|
||||
executeTest("gzip -df ${strFileGz}");
|
||||
$self->testResult(
|
||||
sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031',
|
||||
' check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = undef;
|
||||
|
||||
my $oFile = $self->testResult(
|
||||
sub {storageTest()->openWrite($strFile)}, '[object]', 'open file to extend during compression');
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {lCompressBufferMax => 4194304})},
|
||||
'[object]', ' new read compress');
|
||||
|
||||
$self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write first block');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read compressed first block (compression done)');
|
||||
|
||||
$self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write second block');
|
||||
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000)}, 0, ' read compressed = 0');
|
||||
|
||||
$self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content');
|
||||
executeTest("gzip -df ${strFileGz}");
|
||||
$self->testResult(
|
||||
sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFileGz, $strFileContent);
|
||||
|
||||
$oGzipIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFileGz), {strCompressType => STORAGE_DECOMPRESS})},
|
||||
'[object]', 'new read decompress');
|
||||
|
||||
$self->testException(
|
||||
sub {$oGzipIo->read(\$tBuffer, 1)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}': incorrect header check");
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,107 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Tests for StorageFilterSha module
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageFilterShaPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Test data
|
||||
my $strFile = $self->testPath() . qw{/} . 'file.txt';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileLength = length($strFileContent);
|
||||
my $oDriver = new pgBackRest::Storage::Posix::Driver();
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('read()'))
|
||||
{
|
||||
my $tBuffer;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
|
||||
|
||||
my $oFileIo = $self->testResult(sub {$oDriver->openRead($strFile)}, '[object]', 'open read');
|
||||
my $oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read');
|
||||
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, undef)}, 2, 'read 2 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 2)}, 2, 'read 2 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 4)}, 2, 'read 2 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 6)}, 2, 'read 2 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 8)}, 0, 'read 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oShaIo->close()}, true, 'close');
|
||||
my $strSha = $self->testResult(
|
||||
sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent),
|
||||
'check hash against original content');
|
||||
$self->testResult($strSha, cryptoHashOne('sha1', $tBuffer), 'check hash against buffer');
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$tBuffer = undef;
|
||||
|
||||
$oFileIo = $self->testResult(
|
||||
sub {$oDriver->openRead($self->dataPath() . '/filecopy.archive2.bin')}, '[object]', 'open read');
|
||||
$oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read');
|
||||
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
|
||||
$self->testResult(sub {$oShaIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oShaIo->close()}, true, ' close');
|
||||
$self->testResult(sub {$oShaIo->close()}, false, ' close again to make sure nothing bad happens');
|
||||
$self->testResult($oShaIo->result(STORAGE_FILTER_SHA), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check hash');
|
||||
$self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('write()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oFileIo = $self->testResult(sub {$oDriver->openWrite($strFile, {bAtomic => true})}, '[object]', 'open write');
|
||||
my $oShaIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new');
|
||||
|
||||
my $tBuffer = substr($strFileContent, 0, 2);
|
||||
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 2, 2);
|
||||
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 4, 2);
|
||||
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
|
||||
$tBuffer = substr($strFileContent, 6, 2);
|
||||
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
|
||||
$tBuffer = '';
|
||||
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 0, 'write 0 bytes');
|
||||
|
||||
$self->testResult(sub {$oShaIo->close()}, true, 'close');
|
||||
my $strSha = $self->testResult(
|
||||
sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent),
|
||||
'check hash against original content');
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -61,11 +61,12 @@ sub run
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
if ($self->begin("storageLocal()"))
|
||||
{
|
||||
$self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put');
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put');
|
||||
$self->testResult(sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put');
|
||||
$self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put');
|
||||
|
||||
$self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put cache storage');
|
||||
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put');
|
||||
$self->testResult(
|
||||
sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put cache storage');
|
||||
$self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put');
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
@ -109,44 +110,6 @@ sub run
|
||||
$self->testResult(
|
||||
sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP . '/file')}, $self->testPath() . '/repo/backup/db/file',
|
||||
'check backup file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Insert a bogus rule to generate an error
|
||||
storageRepo()->{hRule}{'<BOGUS>'} =
|
||||
{
|
||||
fnRule => storageRepo()->{hRule}{&STORAGE_REPO_ARCHIVE}{fnRule},
|
||||
};
|
||||
|
||||
$self->testException(sub {storageRepo()->pathGet('<BOGUS>')}, ERROR_ASSERT, 'invalid <REPO> storage rule <BOGUS>');
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
if ($self->begin("storageRepo() encryption"))
|
||||
{
|
||||
my $strStanzaEncrypt = 'test-encrypt';
|
||||
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
|
||||
$self->testException(
|
||||
sub {$self->configTestLoad(CFGCMD_ARCHIVE_PUSH)}, ERROR_OPTION_REQUIRED,
|
||||
'archive-push command requires option: repo1-cipher-pass');
|
||||
|
||||
# Set the encryption passphrase and confirm passphrase and type have been set in the storage object
|
||||
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x');
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
|
||||
$self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherType() eq
|
||||
CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC}, true, 'encryption type set');
|
||||
$self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherPassUser() eq 'x'}, true,
|
||||
'encryption passphrase set');
|
||||
|
||||
# Cannot change encryption after it has been set (cached values not reset)
|
||||
$self->optionTestClear(CFGOPT_REPO_CIPHER_TYPE);
|
||||
$self->optionTestClear(CFGOPT_REPO_CIPHER_PASS);
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
|
||||
$self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherType() eq
|
||||
CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC}, true, 'encryption type not reset');
|
||||
$self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherPassUser() eq 'x'}, true,
|
||||
'encryption passphrase not reset');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,475 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Tests for Storage::Local module
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageLocalPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::Filter::Sha;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Local;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# initModule - common objects and variables used by all tests.
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Local path
|
||||
$self->{strPathLocal} = $self->testPath() . '/local';
|
||||
|
||||
# Create the dynamic rule
|
||||
my $fnRule = sub
|
||||
{
|
||||
my $strRule = shift;
|
||||
my $strFile = shift;
|
||||
my $xData = shift;
|
||||
|
||||
if ($strRule eq '<fn-rule-1>')
|
||||
{
|
||||
return "fn-rule-1/${xData}" . (defined($strFile) ? "/${strFile}" : '');
|
||||
}
|
||||
else
|
||||
{
|
||||
return 'fn-rule-2/' . (defined($strFile) ? "${strFile}/${strFile}" : 'no-file');
|
||||
}
|
||||
};
|
||||
|
||||
# Create the rule hash
|
||||
my $hRule =
|
||||
{
|
||||
'<static-rule>' => 'static-rule-path',
|
||||
'<fn-rule-1>' =>
|
||||
{
|
||||
fnRule => $fnRule,
|
||||
xData => 'test',
|
||||
},
|
||||
'<fn-rule-2>' =>
|
||||
{
|
||||
fnRule => $fnRule,
|
||||
},
|
||||
};
|
||||
|
||||
# Create local storage
|
||||
$self->{oStorageLocal} = new pgBackRest::Storage::Local(
|
||||
$self->pathLocal(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule, bAllowTemp => false});
|
||||
|
||||
# Create encrypted storage
|
||||
$self->{oStorageEncrypt} = new pgBackRest::Storage::Local(
|
||||
$self->testPath(), new pgBackRest::Storage::Posix::Driver(),
|
||||
{hRule => $hRule, bAllowTemp => false, strCipherType => CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC});
|
||||
|
||||
# Remote path
|
||||
$self->{strPathRemote} = $self->testPath() . '/remote';
|
||||
|
||||
# Create the repo path so the remote won't complain that it's missing
|
||||
mkdir($self->pathRemote())
|
||||
or confess &log(ERROR, "unable to create repo directory '" . $self->pathRemote() . qw{'});
|
||||
|
||||
# Remove repo path now that the remote is created
|
||||
rmdir($self->{strPathRemote})
|
||||
or confess &log(ERROR, "unable to remove repo directory '" . $self->pathRemote() . qw{'});
|
||||
|
||||
# Create remote storage
|
||||
$self->{oStorageRemote} = new pgBackRest::Storage::Local(
|
||||
$self->pathRemote(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule});
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# initTest - initialization before each test
|
||||
####################################################################################################################################
|
||||
sub initTest
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
executeTest(
|
||||
'ssh ' . $self->backrestUser() . '\@' . $self->host() . ' mkdir -m 700 ' . $self->pathRemote(), {bSuppressStdErr => true});
|
||||
|
||||
executeTest('mkdir -m 700 ' . $self->pathLocal());
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Define test file
|
||||
my $strFile = 'file.txt';
|
||||
my $strFileCopy = 'file.txt.copy';
|
||||
my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileSize = length($strFileContent);
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("pathGet()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->pathGet('<static-rule>/test', {bTemp => true})},
|
||||
ERROR_ASSERT, "temp file not supported for storage '" . $self->storageLocal()->pathBase() . "'");
|
||||
$self->testException(
|
||||
sub {$self->storageRemote()->pathGet('<static-rule>', {bTemp => true})},
|
||||
ERROR_ASSERT, 'file part must be defined when temp file specified');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageRemote()->pathGet('/file', {bTemp => true})}, "/file.tmp", 'absolute path temp');
|
||||
$self->testResult(sub {$self->storageRemote()->pathGet('/file')}, "/file", 'absolute path file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('file')}, $self->storageLocal()->pathBase() . '/file', 'relative path');
|
||||
$self->testResult(
|
||||
sub {$self->storageRemote()->pathGet('file', {bTemp => true})},
|
||||
$self->storageRemote()->pathBase() . '/file.tmp', 'relative path temp');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->pathGet('<static-rule/file')}, ERROR_ASSERT, "found < but not > in '<static-rule/file'");
|
||||
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->pathGet('<bogus-rule>')}, ERROR_ASSERT, "storage rule '<bogus-rule>' does not exist");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<static-rule>/file')},
|
||||
$self->storageLocal()->pathBase() . '/static-rule-path/file', 'static rule file');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<static-rule>')},
|
||||
$self->storageLocal()->pathBase() . '/static-rule-path', 'static rule path');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<fn-rule-1>/file')},
|
||||
$self->storageLocal()->pathBase() . '/fn-rule-1/test/file', 'function rule 1 file');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<fn-rule-2>/file')},
|
||||
$self->storageLocal()->pathBase() . '/fn-rule-2/file/file', 'function rule 2 file');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<fn-rule-1>')},
|
||||
$self->storageLocal()->pathBase() . '/fn-rule-1/test', 'function rule 1 path');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->pathGet('<fn-rule-2>')},
|
||||
$self->storageLocal()->pathBase() . '/fn-rule-2/no-file', 'function rule 2 no file');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openWrite()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oFileIo = $self->testResult(sub {$self->storageLocal()->openWrite($strFile)}, '[object]', 'open write');
|
||||
|
||||
$self->testResult(sub {$oFileIo->write(\$strFileContent)}, $iFileSize, "write $iFileSize bytes");
|
||||
$self->testResult(sub {$oFileIo->close()}, true, 'close');
|
||||
|
||||
# Check that it is not encrypted
|
||||
$self->testResult(sub {$self->storageLocal()->encrypted($strFile)}, false, 'test storage not encrypted');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('put()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize,
|
||||
'put reference');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openRead()'))
|
||||
{
|
||||
my $tContent;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true})}, undef, 'ignore missing');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_MISSING,
|
||||
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest('sudo touch ' . $self->pathLocal() . "/${strFile} && sudo chmod 700 " . $self->pathLocal() . "/${strFile}");
|
||||
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_OPEN,
|
||||
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': Permission denied");
|
||||
|
||||
executeTest('sudo rm ' . $self->pathLocal() . "/${strFile}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent);
|
||||
|
||||
my $oFileIo = $self->testResult(sub {$self->storageLocal()->openRead($strFile)}, '[object]', 'open read');
|
||||
|
||||
$self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes");
|
||||
$self->testResult($tContent, $strFileContent, ' check read');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileIo = $self->testResult(
|
||||
sub {$self->storageLocal()->openRead($strFile, {rhyFilter => [{strClass => STORAGE_FILTER_SHA}]})}, '[object]',
|
||||
'open read + checksum');
|
||||
|
||||
undef($tContent);
|
||||
$self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes");
|
||||
$self->testResult(sub {$oFileIo->close()}, true, 'close');
|
||||
$self->testResult($tContent, $strFileContent, ' check read');
|
||||
$self->testResult($oFileIo->result(STORAGE_FILTER_SHA), cryptoHashOne('sha1', $strFileContent), ' check hash');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('get()'))
|
||||
{
|
||||
my $tBuffer;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef,
|
||||
'get missing');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile);
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile, $strFileContent);
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('hashSize()'))
|
||||
{
|
||||
my $tBuffer;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put');
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->hashSize($strFile)},
|
||||
qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->hashSize(BOGUS, {bIgnoreMissing => true})}, "([undef], [undef])",
|
||||
' check missing hash/size');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('copy()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
|
||||
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy(
|
||||
$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}),
|
||||
$self->storageLocal()->openWrite($strFileCopy))},
|
||||
false, 'missing source io');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
|
||||
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile, $strFileContent);
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->remove($strFileCopy);
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->remove($strFileCopy);
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy(
|
||||
$self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))},
|
||||
true, 'copy io->io');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('info()'))
|
||||
{
|
||||
$self->testResult(sub {$self->storageLocal()->info($self->{strPathLocal})}, "[object]", 'stat dir successfully');
|
||||
|
||||
$self->testException(sub {$self->storageLocal()->info($strFile)}, ERROR_FILE_MISSING,
|
||||
"unable to stat '". $self->{strPathLocal} . "/" . $strFile ."': No such file or directory");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('pathCreate()'))
|
||||
{
|
||||
my $strTestPath = $self->{strPathLocal} . "/" . BOGUS;
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]",
|
||||
"test creation of path " . $strTestPath);
|
||||
|
||||
$self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_EXISTS,
|
||||
"unable to create path '". $strTestPath. "' because it already exists");
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]",
|
||||
"ignore path exists");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('encryption'))
|
||||
{
|
||||
my $strCipherPass = 'x';
|
||||
$self->testResult(sub {cryptoHashOne('sha1', $strFileContent)}, $strFileHash, 'hash check contents to be written');
|
||||
|
||||
# Error when passphrase not passed
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oFileIo = $self->testException(sub {$self->storageEncrypt()->openWrite($strFile)},
|
||||
ERROR_ASSERT, 'tCipherPass is required in Storage::Filter::CipherBlock->new');
|
||||
|
||||
# Write an encrypted file
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileIo = $self->testResult(sub {$self->storageEncrypt()->openWrite($strFile, {strCipherPass => $strCipherPass})},
|
||||
'[object]', 'open write');
|
||||
|
||||
my $iWritten = $oFileIo->write(\$strFileContent);
|
||||
$self->testResult(sub {$oFileIo->close()}, true, ' close');
|
||||
|
||||
# Check that it is encrypted and valid for the repo encryption type
|
||||
$self->testResult(sub {$self->storageEncrypt()->encryptionValid($self->storageEncrypt()->encrypted($strFile))}, true,
|
||||
' test storage encrypted and valid');
|
||||
|
||||
$self->testResult(
|
||||
sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)}) ne $strFileHash}, true, ' check written sha1 different');
|
||||
|
||||
# Error when passphrase not passed
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileIo = $self->testException(sub {$self->storageEncrypt()->openRead($strFile)},
|
||||
ERROR_ASSERT, 'tCipherPass is required in Storage::Filter::CipherBlock->new');
|
||||
|
||||
# Read it and confirm it decrypts and is same as original content
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileIo = $self->testResult(sub {$self->storageEncrypt()->openRead($strFile, {strCipherPass => $strCipherPass})},
|
||||
'[object]', 'open read and decrypt');
|
||||
my $strContent;
|
||||
$oFileIo->read(\$strContent, $iWritten);
|
||||
$self->testResult(sub {$oFileIo->close()}, true, ' close');
|
||||
$self->testResult($strContent, $strFileContent, ' decrypt read equal orginal contents');
|
||||
|
||||
# Copy
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageEncrypt()->copy(
|
||||
$self->storageEncrypt()->openRead($strFile, {strCipherPass => $strCipherPass}),
|
||||
$self->storageEncrypt()->openWrite($strFileCopy, {strCipherPass => $strCipherPass}))},
|
||||
true, 'copy - decrypt/encrypt');
|
||||
|
||||
$self->testResult(
|
||||
sub {cryptoHashOne('sha1', ${$self->storageEncrypt()->get($strFileCopy, {strCipherPass => $strCipherPass})})},
|
||||
$strFileHash, ' check decrypted copy file sha1 same as original plaintext file');
|
||||
|
||||
# Write an empty encrypted file
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strFileZero = 'file-0.txt';
|
||||
my $strZeroContent = '';
|
||||
$oFileIo = $self->testResult(
|
||||
sub {$self->storageEncrypt()->openWrite($strFileZero, {strCipherPass => $strCipherPass})}, '[object]',
|
||||
'open write for zero');
|
||||
|
||||
$self->testResult(sub {$oFileIo->write(\$strZeroContent)}, 0, ' zero written');
|
||||
$self->testResult(sub {$oFileIo->close()}, true, ' close');
|
||||
|
||||
$self->testResult(sub {$self->storageEncrypt()->encrypted($strFile)}, true, ' test empty file encrypted');
|
||||
|
||||
# Write an unencrypted file to the encrypted storage and check if the file is valid for that storage
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strFileTest = $self->testPath() . qw{/} . 'test.file.txt';
|
||||
|
||||
# Create empty file
|
||||
executeTest("touch ${strFileTest}");
|
||||
$self->testResult(sub {$self->storageEncrypt()->encrypted($strFileTest)}, false, 'empty file so not encrypted');
|
||||
|
||||
# Add unencrypted content to the file
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFileTest}");
|
||||
$self->testResult(sub {$self->storageEncrypt()->encryptionValid($self->storageEncrypt()->encrypted($strFileTest))}, false,
|
||||
'storage encryption and unencrypted file format do not match');
|
||||
|
||||
# Unencrypted file valid in unencrypted storage
|
||||
$self->testResult(sub {$self->storageLocal()->encryptionValid($self->storageLocal()->encrypted($strFileTest))}, true,
|
||||
'unencrypted file valid in unencrypted storage');
|
||||
|
||||
# Prepend encryption Magic Signature and test encrypted file in unencrypted storage not valid
|
||||
executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strFileTest . ')" > ' . $strFileTest);
|
||||
$self->testResult(sub {$self->storageLocal()->encryptionValid($self->storageLocal()->encrypted($strFileTest))}, false,
|
||||
'storage unencrypted and encrypted file format do not match');
|
||||
|
||||
# Test a file that does not exist
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strFileTest = $self->testPath() . qw{/} . 'testfile';
|
||||
$self->testException(sub {$self->storageEncrypt()->encrypted($strFileTest)}, ERROR_FILE_MISSING,
|
||||
"unable to open '" . $strFileTest . "': No such file or directory");
|
||||
|
||||
$self->testResult(sub {$self->storageEncrypt()->encrypted($strFileTest, {bIgnoreMissing => true})}, true,
|
||||
'encryption for ignore missing file returns encrypted for encrypted storage');
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->encrypted($strFileTest, {bIgnoreMissing => true})}, false,
|
||||
'encryption for ignore missing file returns unencrypted for unencrypted storage');
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
sub host {return '127.0.0.1'}
|
||||
sub pathLocal {return shift->{strPathLocal}};
|
||||
sub pathRemote {return shift->{strPathRemote}};
|
||||
sub protocolLocal {return shift->{oProtocolLocal}};
|
||||
sub protocolRemote {return shift->{oProtocolRemote}};
|
||||
sub storageLocal {return shift->{oStorageLocal}};
|
||||
sub storageEncrypt {return shift->{oStorageEncrypt}};
|
||||
sub storageRemote {return shift->{oStorageRemote}};
|
||||
|
||||
1;
|
||||
341
test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm
Normal file
341
test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm
Normal file
@ -0,0 +1,341 @@
|
||||
####################################################################################################################################
|
||||
# Tests for Storage::Local module
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StoragePerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::Base;
|
||||
|
||||
use pgBackRestTest::Common::ContainerTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Define test file
|
||||
my $strFile = $self->testPath() . '/file.txt';
|
||||
my $strFileCopy = $self->testPath() . '/file.txt.copy';
|
||||
my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileSize = length($strFileContent);
|
||||
|
||||
# Create local storage
|
||||
$self->{oStorageLocal} = new pgBackRest::Storage::Storage('<LOCAL>');
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("pathGet()"))
|
||||
{
|
||||
$self->testResult(sub {$self->storageLocal()->pathGet('file')}, '/file', 'relative path');
|
||||
$self->testResult(sub {$self->storageLocal()->pathGet('/file2')}, '/file2', 'absolute path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('put()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize,
|
||||
'put reference');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('get()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef,
|
||||
'get missing');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile);
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile, $strFileContent);
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('hashSize()'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put');
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->hashSize($strFile)},
|
||||
qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->hashSize(BOGUS, {bIgnoreMissing => true})}, "([undef], [undef])",
|
||||
' check missing hash/size');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('copy()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
|
||||
"unable to open missing file '${strFile}' for read");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy(
|
||||
$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}),
|
||||
$self->storageLocal()->openWrite($strFileCopy))},
|
||||
false, 'missing source io');
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
|
||||
"unable to open missing file '${strFile}' for read");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->put($strFile, $strFileContent);
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->remove($strFileCopy);
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->storageLocal()->remove($strFileCopy);
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->copy(
|
||||
$self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))},
|
||||
true, 'copy io->io');
|
||||
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('exists()'))
|
||||
{
|
||||
$self->storageLocal()->put($self->testPath() . "/test.file");
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.file")}, true, 'existing file');
|
||||
$self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.missing")}, false, 'missing file');
|
||||
$self->testResult(sub {$self->storageLocal()->exists($self->testPath())}, false, 'path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('info()'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->info($self->testPath())},
|
||||
"{group => " . $self->group() . ", mode => 0770, type => d, user => " . $self->pgUser() . "}",
|
||||
'stat dir successfully');
|
||||
|
||||
$self->testException(sub {$self->storageLocal()->info(BOGUS)}, ERROR_FILE_OPEN,
|
||||
"unable to get info for missing path/file '/bogus'");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("manifest() and list()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->manifest($self->testPath() . '/missing')},
|
||||
ERROR_PATH_MISSING, "unable to list file info for missing path '" . $self->testPath() . "/missing'");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Setup test data
|
||||
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1');
|
||||
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2');
|
||||
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub2');
|
||||
|
||||
executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt');
|
||||
utime(1111111111, 1111111111, $self->testPath() . '/test.txt');
|
||||
executeTest('chmod 1640 ' . $self->testPath() . '/test.txt');
|
||||
|
||||
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt');
|
||||
utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt');
|
||||
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt');
|
||||
|
||||
executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt');
|
||||
utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt');
|
||||
executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt');
|
||||
|
||||
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt');
|
||||
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt');
|
||||
|
||||
executeTest('ln -s .. ' . $self->testPath() . '/sub1/test');
|
||||
executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test');
|
||||
executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test');
|
||||
executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test');
|
||||
|
||||
executeTest('chmod 0770 ' . $self->testPath());
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->manifest($self->testPath())},
|
||||
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test-hardlink.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test-sub2.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/test-hardlink.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/test-sub1.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'test.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}}',
|
||||
'complete manifest');
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->list($self->testPath())}, "(sub1, sub2, test.txt)", "list");
|
||||
$self->testResult(sub {$self->storageLocal()->list($self->testPath(), {strExpression => "2\$"})}, "sub2", "list");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->list($self->testPath(), {strSortOrder => 'reverse'})}, "(test.txt, sub2, sub1)",
|
||||
"list reverse");
|
||||
$self->testResult(sub {$self->storageLocal()->list($self->testPath() . "/sub2")}, "[undef]", "list empty");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->list($self->testPath() . "/sub99", {bIgnoreMissing => true})}, "[undef]", "list missing");
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->list($self->testPath() . "/sub99")}, ERROR_PATH_MISSING,
|
||||
"unable to list files for missing path '" . $self->testPath() . "/sub99'");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('move()'))
|
||||
{
|
||||
my $strFileCopy = "${strFile}.copy";
|
||||
my $strFileSub = $self->testPath() . '/sub/file.txt';
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->move($strFile, $strFileCopy)}, ERROR_FILE_MOVE,
|
||||
"unable to move '${strFile}' to '${strFile}.copy': No such file or directory");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('owner()'))
|
||||
{
|
||||
my $strFile = $self->testPath() . "/test.txt";
|
||||
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->owner($strFile, 'root')}, ERROR_FILE_MISSING,
|
||||
"unable to stat '${strFile}': No such file or directory");
|
||||
|
||||
executeTest("touch ${strFile}");
|
||||
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->owner($strFile, BOGUS)}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}' because user 'bogus' does not exist");
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->owner($strFile, undef, BOGUS)}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}' because group 'bogus' does not exist");
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->owner($strFile)}, undef, "no ownership changes");
|
||||
$self->testResult(sub {$self->storageLocal()->owner($strFile, TEST_USER)}, undef, "same user");
|
||||
$self->testResult(sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "same group");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->owner($strFile, TEST_USER, TEST_GROUP)}, undef,
|
||||
"same user, group");
|
||||
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->owner($strFile, 'root', undef)}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}': Operation not permitted");
|
||||
$self->testException(
|
||||
sub {$self->storageLocal()->owner($strFile, undef, 'root')}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}': Operation not permitted");
|
||||
|
||||
executeTest("sudo chown :root ${strFile}");
|
||||
$self->testResult(
|
||||
sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "change group back from root");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('pathCreate()'))
|
||||
{
|
||||
my $strTestPath = $self->testPath() . "/" . BOGUS;
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]",
|
||||
"test creation of path " . $strTestPath);
|
||||
|
||||
$self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_CREATE,
|
||||
"unable to create path '". $strTestPath. "'");
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]",
|
||||
"ignore path exists");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('pathExists()'))
|
||||
{
|
||||
$self->storageLocal()->put($self->testPath() . "/test.file");
|
||||
|
||||
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.file")}, false, 'existing file');
|
||||
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.missing")}, false, 'missing file');
|
||||
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath())}, true, 'path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('pathSync()'))
|
||||
{
|
||||
$self->testResult(sub {$self->storageLocal()->pathSync($self->testPath())}, "[undef]", "test path sync");
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Getters
|
||||
####################################################################################################################################
|
||||
# sub host {return '127.0.0.1'}
|
||||
# sub pathLocal {return shift->{strPathLocal}};
|
||||
# sub pathRemote {return shift->{strPathRemote}};
|
||||
sub storageLocal {return shift->{oStorageLocal}};
|
||||
# sub storageEncrypt {return shift->{oStorageEncrypt}};
|
||||
# sub storageRemote {return shift->{oStorageRemote}};
|
||||
|
||||
1;
|
||||
@ -1,444 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# Posix Driver Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StoragePosixPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use File::Basename qw(basename dirname);
|
||||
use IO::Socket::UNIX;
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Storage::Posix::Driver;
|
||||
|
||||
use pgBackRestTest::Common::ContainerTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Test data
|
||||
my $strFile = $self->testPath() . '/file.txt';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileLength = length($strFileContent);
|
||||
my $iFileLengthHalf = int($iFileLength / 2);
|
||||
|
||||
# Test driver
|
||||
my $oPosix = new pgBackRest::Storage::Posix::Driver();
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('exists()'))
|
||||
{
|
||||
my $strPathSub = $self->testPath() . '/sub';
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oPosix->exists($strFile)}, false, 'file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("sudo mkdir ${strPathSub} && sudo chmod 700 ${strPathSub}");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->pathExists($strPathSub)}, true, 'path');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$oPosix->exists("${strPathSub}/file")}, ERROR_FILE_EXISTS,
|
||||
"unable to test if file '${strPathSub}/file' exists: Permission denied");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("manifestList()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my @stryFile = ('.', 'test.txt');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->manifestList($self->testPath(), \@stryFile)},
|
||||
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}}',
|
||||
'skip missing file');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("manifestStat()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strFile = $self->testPath() . '/test.txt';
|
||||
|
||||
$self->testResult(sub {$oPosix->manifestStat($strFile)}, '[undef]', 'ignore missing file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFile, "TEST");
|
||||
utime(1111111111, 1111111111, $strFile);
|
||||
executeTest('chmod 1640 ' . $strFile);
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->manifestStat($strFile)},
|
||||
'{group => ' . $self->group() .
|
||||
', mode => 1640, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}',
|
||||
'stat file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strSocketFile = $self->testPath() . '/test.socket';
|
||||
|
||||
# Create a socket to test invalid files
|
||||
my $oSocket = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->manifestStat($strSocketFile)}, ERROR_FILE_INVALID,
|
||||
"${strSocketFile} is not of type directory, file, or link");
|
||||
|
||||
# Cleanup socket
|
||||
$oSocket->close();
|
||||
storageTest()->remove($strSocketFile);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strTestPath = $self->testPath() . '/public_dir';
|
||||
storageTest()->pathCreate($strTestPath, {strMode => '0750'});
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->manifestStat($strTestPath)},
|
||||
'{group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}',
|
||||
'stat directory');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strTestLink = $self->testPath() . '/public_dir_link';
|
||||
|
||||
symlink($strTestPath, $strTestLink)
|
||||
or confess &log(ERROR, "unable to create symlink from ${strTestPath} to ${strTestLink}");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->manifestStat($strTestLink)},
|
||||
'{group => ' . $self->group() . ", link_destination => ${strTestPath}, type => l, user => " . $self->pgUser() . '}',
|
||||
'stat link');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("manifestRecurse()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strTestPath = $self->testPath() . '/public_dir';
|
||||
my $strTestFile = "${strTestPath}/test.txt";
|
||||
|
||||
$self->testException(
|
||||
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
|
||||
ERROR_FILE_MISSING, "unable to stat '${strTestFile}': No such file or directory");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->pathCreate($strTestPath, {strMode => '0750'});
|
||||
|
||||
$self->testResult(
|
||||
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestPath, undef, 0, $hManifest); $hManifest},
|
||||
'{. => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}}',
|
||||
'empty directory manifest');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strTestFile, "TEST");
|
||||
utime(1111111111, 1111111111, $strTestFile);
|
||||
executeTest('chmod 0750 ' . $strTestFile);
|
||||
|
||||
storageTest()->pathCreate("${strTestPath}/sub", {strMode => '0750'});
|
||||
|
||||
$self->testResult(
|
||||
sub {my $hManifest = {}; $oPosix->manifestRecurse(
|
||||
$self->testPath(), basename($strTestPath), 1, $hManifest); $hManifest},
|
||||
'{public_dir => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'public_dir/sub => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'public_dir/' . basename($strTestFile) . ' => {group => ' . $self->group() .
|
||||
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
|
||||
'directory and file manifest');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
|
||||
'{' . basename($strTestFile) . ' => {group => ' . $self->group() .
|
||||
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
|
||||
'single file manifest');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("manifest()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strMissingFile = $self->testPath() . '/missing';
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->manifest($strMissingFile)},
|
||||
ERROR_FILE_MISSING, "unable to stat '${strMissingFile}': No such file or directory");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Setup test data
|
||||
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1');
|
||||
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2');
|
||||
|
||||
executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt');
|
||||
utime(1111111111, 1111111111, $self->testPath() . '/test.txt');
|
||||
executeTest('chmod 1640 ' . $self->testPath() . '/test.txt');
|
||||
|
||||
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt');
|
||||
utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt');
|
||||
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt');
|
||||
|
||||
executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt');
|
||||
utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt');
|
||||
executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt');
|
||||
|
||||
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt');
|
||||
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt');
|
||||
|
||||
executeTest('ln -s .. ' . $self->testPath() . '/sub1/test');
|
||||
executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test');
|
||||
executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test');
|
||||
executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test');
|
||||
|
||||
executeTest('chmod 0770 ' . $self->testPath());
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->manifest($self->testPath())},
|
||||
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test-hardlink.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/sub2/test-sub2.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' .
|
||||
'sub1/test-hardlink.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'sub1/test-sub1.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' .
|
||||
$self->pgUser() . '}, ' .
|
||||
'test.txt => ' .
|
||||
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
|
||||
$self->pgUser() . '}}',
|
||||
'complete manifest');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openRead() & Posix::FileRead'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$oPosix->openRead($strFile)}, ERROR_FILE_MISSING, "unable to open '${strFile}': No such file or directory");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPosix->openRead($strFile)}, '[object]', 'open read');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openWrite() & Posix::FileWrite'))
|
||||
{
|
||||
my $tContent = $strFileContent;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
|
||||
executeTest("chmod 600 ${strFile} && sudo chown root:root ${strFile}");
|
||||
|
||||
$self->testException(
|
||||
sub {new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile)}, ERROR_FILE_OPEN,
|
||||
"unable to open '${strFile}': Permission denied");
|
||||
|
||||
executeTest("sudo rm -rf ${strFile}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oPosixIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open');
|
||||
|
||||
$tContent = undef;
|
||||
$self->testException(
|
||||
sub {$oPosixIo->write(\$tContent)}, ERROR_FILE_WRITE, "unable to write to '${strFile}': Use of uninitialized value");
|
||||
|
||||
$tContent = substr($strFileContent, 0, $iFileLengthHalf);
|
||||
$self->testResult(
|
||||
sub {$oPosixIo->write(\$tContent)}, $iFileLengthHalf, 'write part 1');
|
||||
|
||||
$tContent = substr($strFileContent, $iFileLengthHalf);
|
||||
$self->testResult(
|
||||
sub {$oPosixIo->write(\$tContent)}, $iFileLength - $iFileLengthHalf,
|
||||
'write part 2');
|
||||
$oPosixIo->close();
|
||||
|
||||
$tContent = undef;
|
||||
$self->testResult(
|
||||
sub {(new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile))->read(\$tContent, $iFileLength)},
|
||||
$iFileLength, 'check write content length');
|
||||
$self->testResult($tContent, $strFileContent, 'check write content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oPosixIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Posix::FileWrite(
|
||||
$oPosix, "${strFile}.atomic", {bAtomic => true, strMode => '0666', lTimestamp => time(), bSync => false})},
|
||||
'[object]', 'open');
|
||||
|
||||
$self->testResult(sub {$oPosixIo->write(\$tContent, $iFileLength)}, $iFileLength, 'write');
|
||||
$self->testResult(sub {$oPosixIo->close()}, true, 'close');
|
||||
|
||||
$self->testResult(sub {${storageTest()->get("${strFile}.atomic")}}, $strFileContent, 'check content');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oPosixIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open');
|
||||
|
||||
$self->testResult(sub {$oPosixIo->close()}, true, 'close');
|
||||
|
||||
undef($oPosixIo);
|
||||
|
||||
# Test that a premature destroy (from error or otherwise) does not rename the file
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strFileAbort = $self->testPath() . '/file-abort.txt';
|
||||
my $strFileAbortTmp = "${strFileAbort}.tmp";
|
||||
|
||||
$oPosixIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFileAbort, {bAtomic => true})}, '[object]', 'open');
|
||||
|
||||
$oPosixIo->write(\$strFileContent);
|
||||
undef($oPosixIo);
|
||||
|
||||
$self->testResult(sub {$oPosix->exists($strFileAbort)}, false, 'destination file does not exist');
|
||||
$self->testResult(sub {$oPosix->exists($strFileAbortTmp)}, true, 'destination file tmp exists');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oPosixIo = $self->testResult(
|
||||
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile, {lTimestamp => time()})}, '[object]', 'open');
|
||||
$self->testResult(sub {$oPosixIo->write(\$strFileContent, $iFileLength)}, $iFileLength, 'write');
|
||||
executeTest("rm -f $strFile");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosixIo->close()}, ERROR_FILE_WRITE, "unable to set time for '${strFile}': No such file or directory");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('owner()'))
|
||||
{
|
||||
my $strFile = $self->testPath() . "/test.txt";
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->owner($strFile, {strUser => 'root'})}, ERROR_FILE_MISSING,
|
||||
"unable to stat '${strFile}': No such file or directory");
|
||||
|
||||
executeTest("touch ${strFile}");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->owner($strFile, {strUser => BOGUS})}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}' because user 'bogus' does not exist");
|
||||
$self->testException(
|
||||
sub {$oPosix->owner($strFile, {strGroup => BOGUS})}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}' because group 'bogus' does not exist");
|
||||
|
||||
$self->testResult(sub {$oPosix->owner($strFile)}, undef, "no ownership changes");
|
||||
$self->testResult(sub {$oPosix->owner($strFile, {strUser => TEST_USER})}, undef, "same user");
|
||||
$self->testResult(sub {$oPosix->owner($strFile, {strGroup => TEST_GROUP})}, undef, "same group");
|
||||
$self->testResult(
|
||||
sub {$oPosix->owner($strFile, {strUser => TEST_USER, strGroup => TEST_GROUP})}, undef, "same user, group");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->owner($strFile, {strUser => 'root'})}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}': Operation not permitted");
|
||||
$self->testException(
|
||||
sub {$oPosix->owner($strFile, {strGroup => 'root'})}, ERROR_FILE_OWNER,
|
||||
"unable to set ownership for '${strFile}': Operation not permitted");
|
||||
|
||||
executeTest("sudo chown :root ${strFile}");
|
||||
$self->testResult(
|
||||
sub {$oPosix->owner($strFile, {strGroup => TEST_GROUP})}, undef, "change group back from root");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('pathCreate()'))
|
||||
{
|
||||
my $strPathParent = $self->testPath() . '/parent';
|
||||
my $strPathSub = "${strPathParent}/sub1/sub2";
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oPosix->pathCreate($strPathParent)}, undef, 'parent path');
|
||||
$self->testResult(
|
||||
sub {$oPosix->pathExists($strPathParent)}, true, ' check path');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$oPosix->pathCreate($strPathParent)}, ERROR_PATH_EXISTS,
|
||||
"unable to create path '${strPathParent}' because it already exists");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oPosix->pathCreate($strPathParent, {bIgnoreExists => true})}, undef, 'path already exists');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("sudo chown root:root ${strPathParent} && sudo chmod 700 ${strPathParent}");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_CREATE,
|
||||
"unable to create path '${strPathSub}': Permission denied");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("rmdir ${strPathParent}");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_MISSING,
|
||||
"unable to create path '${strPathSub}' because parent does not exist");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oPosix->pathCreate($strPathSub, {bCreateParent => true})}, undef, 'path with parents');
|
||||
$self->testResult(
|
||||
sub {$oPosix->pathExists($strPathSub)}, true, ' check path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('move()'))
|
||||
{
|
||||
my $strFileCopy = "${strFile}.copy";
|
||||
my $strFileSub = $self->testPath() . '/sub/file.txt';
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$oPosix->move($strFile, $strFileCopy)}, ERROR_FILE_MISSING,
|
||||
"unable to move '${strFile}' because it is missing");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {storageTest()->put($strFile, $strFileContent)}, $iFileLength, 'put');
|
||||
$self->testResult(
|
||||
sub {$oPosix->move($strFile, $strFileCopy)}, undef, 'simple move');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_PATH_MISSING,
|
||||
"unable to move '${strFileCopy}' to missing path '" . dirname($strFileSub) . "'");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest('sudo mkdir ' . dirname($strFileSub) . ' && sudo chmod 700 ' . dirname($strFileSub));
|
||||
|
||||
$self->testException(
|
||||
sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_FILE_MOVE,
|
||||
"unable to move '${strFileCopy}' to '${strFileSub}': Permission denied");
|
||||
|
||||
executeTest('sudo rmdir ' . dirname($strFileSub));
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oPosix->move($strFileCopy, $strFileSub, {bCreatePath => true})}, undef, 'create parent path');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,167 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Authentication Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageS3AuthPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use POSIX qw(strftime);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Storage::S3::Auth;
|
||||
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('s3DateTime'))
|
||||
{
|
||||
$self->testResult(sub {s3DateTime(1491267845)}, '20170404T010405Z', 'format date/time');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
waitRemainder();
|
||||
$self->testResult(sub {s3DateTime()}, strftime("%Y%m%dT%H%M%SZ", gmtime()), 'format current date/time');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('s3CanonicalRequest'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {s3CanonicalRequest(
|
||||
'GET', qw(/), 'list-type=2',
|
||||
{'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z',
|
||||
'x-amz-content-sha256' => '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9'},
|
||||
'705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
|
||||
"(GET\n/\nlist-type=2\nhost:bucket.s3.amazonaws.com\n" .
|
||||
"x-amz-content-sha256:705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9\n" .
|
||||
"x-amz-date:20170606T121212Z\n\nhost;x-amz-content-sha256;x-amz-date\n" .
|
||||
'705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9' .
|
||||
', host;x-amz-content-sha256;x-amz-date)',
|
||||
'canonical request');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {s3CanonicalRequest(
|
||||
'GET', qw(/), 'list-type=2', {'Host' => 'bucket.s3.amazonaws.com'},
|
||||
'705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
|
||||
ERROR_ASSERT, "header 'Host' must be lower case");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('s3SigningKey'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
|
||||
'705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'signing key');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
|
||||
'705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'same signing key from cache');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {unpack('H*', s3SigningKey('20170505', 'us-west-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
|
||||
'c1a1cb590bbc38ba789c8e5695a1ec0cd7fd44c6949f922e149005a221524c09', 'new signing key');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('s3StringToSign'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {s3StringToSign(
|
||||
'20170412T141414Z', 'us-east-1', '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
|
||||
"AWS4-HMAC-SHA256\n20170412T141414Z\n20170412/us-east-1/s3/aws4_request\n" .
|
||||
"705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9",
|
||||
'string to sign');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('s3AuthorizationHeader'))
|
||||
{
|
||||
$self->testResult(
|
||||
sub {s3AuthorizationHeader(
|
||||
'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z',
|
||||
{'authorization' => BOGUS, 'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'},
|
||||
'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', undef,
|
||||
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')},
|
||||
'({authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' .
|
||||
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' .
|
||||
'Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3,' .
|
||||
' host => bucket.s3.amazonaws.com,' .
|
||||
' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' .
|
||||
' x-amz-date => 20170606T121212Z}, ' .
|
||||
"GET\n" .
|
||||
"/\n" .
|
||||
"list-type=2\n" .
|
||||
"host:bucket.s3.amazonaws.com\n" .
|
||||
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" .
|
||||
"x-amz-date:20170606T121212Z\n" .
|
||||
"\n" .
|
||||
"host;x-amz-content-sha256;x-amz-date\n" .
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855, " .
|
||||
"host;x-amz-content-sha256;x-amz-date, " .
|
||||
"AWS4-HMAC-SHA256\n" .
|
||||
"20170606T121212Z\n" .
|
||||
"20170606/us-east-1/s3/aws4_request\n" .
|
||||
"4f2d4ee971f579e60ba6b3895e87434e17b1260f04392f02b512c1e8bada72dd)",
|
||||
'authorization header request');
|
||||
|
||||
$self->testResult(
|
||||
sub {s3AuthorizationHeader(
|
||||
'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z',
|
||||
{'authorization' => BOGUS, 'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'},
|
||||
'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
|
||||
'AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQW' .
|
||||
'LWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGd' .
|
||||
'QrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU' .
|
||||
'9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz' .
|
||||
'+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==',
|
||||
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')},
|
||||
'({authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' .
|
||||
'SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,' .
|
||||
'Signature=c12565bf5d7e0ef623f76d66e09e5431aebef803f6a25a01c586525f17e474a3,' .
|
||||
' host => bucket.s3.amazonaws.com,' .
|
||||
' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' .
|
||||
' x-amz-date => 20170606T121212Z, x-amz-security-token => AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4H' .
|
||||
'IZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZ' .
|
||||
'ampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+sc' .
|
||||
'qKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==}, ' .
|
||||
"GET\n" .
|
||||
"/\n" .
|
||||
"list-type=2\n" .
|
||||
"host:bucket.s3.amazonaws.com\n" .
|
||||
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" .
|
||||
"x-amz-date:20170606T121212Z\n" .
|
||||
"x-amz-security-token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIe" .
|
||||
"oIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlR" .
|
||||
"d8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJab" .
|
||||
"IQwj2ICCR/oLxBA==\n" .
|
||||
"\n" .
|
||||
"host;x-amz-content-sha256;x-amz-date;x-amz-security-token\n" .
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855, " .
|
||||
"host;x-amz-content-sha256;x-amz-date;x-amz-security-token, " .
|
||||
"AWS4-HMAC-SHA256\n" .
|
||||
"20170606T121212Z\n" .
|
||||
"20170606/us-east-1/s3/aws4_request\n" .
|
||||
"c171e7a68355ef4e0e6e1003d2d4a79a7b06e7424e3000ba619f5f7882a3251e)",
|
||||
'authorization header request with token');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,113 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 SSL Certificate Tests
|
||||
#
|
||||
# Verify that SSL certificate validation works on live S3 servers.
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageS3CertPerlTest;
|
||||
use parent 'pgBackRestTest::Env::ConfigEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use Storable qw(dclone);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
use pgBackRestTest::Common::VmTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Use long random string so bucket lookups will fail and expose access errors
|
||||
my $strBucket = 'bnBfyKpXR8ZqQY5RXszxemRgvtmjXd4tf5HkFYhTpT9BndUCYMDy5NCCyRz';
|
||||
my $strEndpoint = 's3-us-west-2.amazonaws.com';
|
||||
my $strRegion = 'us-west-2';
|
||||
|
||||
# Options
|
||||
$self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_KEY, BOGUS);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, BOGUS);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_TOKEN, BOGUS);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_BUCKET, $strBucket);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, $strEndpoint);
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_REGION, $strRegion);
|
||||
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
|
||||
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('validation'))
|
||||
{
|
||||
if ($self->vm eq VM_U12)
|
||||
{
|
||||
&log(INFO, 'cannot test - certificates are no longer maintained for ' . $self->vm());
|
||||
}
|
||||
else
|
||||
{
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
if ($self->vm() eq VM_CO7)
|
||||
{
|
||||
# Tests fails on co7 because by default certs cannot be located. This logic may need to be changed in the future if
|
||||
# this bug gets fixed by Red Hat. UPDATE: The behavior changed here but it does not seems to be fixed.
|
||||
$self->testException(
|
||||
sub {storageRepo({strStanza => 'test1'})->list('/')}, ERROR_HOST_CONNECT,
|
||||
'SSL connect attempt failed with unknown error error.*certificate verify failed',
|
||||
'cert verify fails on ' . VM_CO7);
|
||||
|
||||
# It should work when verification is disabled
|
||||
$self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false);
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
|
||||
$self->testException(
|
||||
sub {storageRepo({strStanza => 'test2'})->list('/')}, ERROR_PROTOCOL, 'S3 request error \[403\] Forbidden.*',
|
||||
'connection succeeds with verification disabled, (expected) error on invalid access key');
|
||||
|
||||
$self->optionTestClear(CFGOPT_REPO_S3_VERIFY_TLS);
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
}
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
# CO7 doesn't locate certs automatically so specify the path
|
||||
if ($self->vm() eq VM_CO7)
|
||||
{
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_CA_FILE, '/etc/pki/tls/certs/ca-bundle.crt');
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
}
|
||||
|
||||
$self->testException(
|
||||
sub {storageRepo({strStanza => 'test3'})->list('/')}, ERROR_PROTOCOL, 'S3 request error \[403\] Forbidden.*',
|
||||
'connection succeeds, (expected) error on invalid access key');
|
||||
|
||||
if ($self->vm() eq VM_CO7)
|
||||
{
|
||||
$self->optionTestClear(CFGOPT_REPO_S3_CA_FILE);
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
}
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$self->optionTestSet(CFGOPT_REPO_S3_CA_PATH, '/bogus');
|
||||
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
|
||||
|
||||
$self->testException(
|
||||
sub {storageRepo({strStanza => 'test4'})->list('/')}, ERROR_HOST_CONNECT,
|
||||
$self->vm() eq VM_CO6 ? 'SSL connect attempt failed with unknown error.*certificate verify failed' : 'No such file or directory',
|
||||
'invalid ca path');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,219 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Storage Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageS3PerlTest;
|
||||
use parent 'pgBackRestTest::Env::S3EnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::String;
|
||||
use pgBackRest::LibC qw(:crypto);
|
||||
use pgBackRest::Storage::S3::Driver;
|
||||
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# initTest
|
||||
####################################################################################################################################
|
||||
sub initTest
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
executeTest("$self->{strS3Command} rm --recursive s3://pgbackrest-dev");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Initialize the driver
|
||||
my $oS3 = $self->initS3();
|
||||
my $oStorage = new pgBackRest::Storage::Local('', $oS3);
|
||||
|
||||
# Test variables
|
||||
my $strFile = 'file.txt';
|
||||
my $strFileContent = 'TESTDATA';
|
||||
my $iFileLength = length($strFileContent);
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('exists()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oStorage->exists($strFile)}, false, 'root file does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFile, $strFileContent);
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
|
||||
|
||||
$self->testResult(sub {$oStorage->exists($strFile)}, true, 'root file exists');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oStorage->pathExists('/path/to')}, false, 'sub path does not exist');
|
||||
$self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, false, 'sub file does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev/path/to/${strFile}");
|
||||
|
||||
$self->testResult(sub {$oStorage->pathExists('/path/to')}, true, 'sub path exists');
|
||||
# $oStorage->pathExists('/path/to');
|
||||
$self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, true, 'sub file exists');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('manifest()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oStorage->manifest('')}, '{. => {type => d}}', 'no files');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFile, $strFileContent);
|
||||
storageTest()->put("${strFile}2", $strFileContent . '2');
|
||||
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oStorage->manifest('')},
|
||||
'{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' .
|
||||
' path/to/file.txt2 => {size => 9, type => f}}',
|
||||
'root path');
|
||||
$self->testResult(
|
||||
sub {$oStorage->manifest('/path/to')}, '{. => {type => d}, file.txt2 => {size => 9, type => f}}', 'sub path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('list()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oStorage->list('')}, '[undef]', 'no files');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFile, $strFileContent);
|
||||
storageTest()->put("${strFile}2", $strFileContent . '2');
|
||||
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
|
||||
|
||||
$self->testResult(sub {$oStorage->list('')}, '(file.txt, path)', 'root path');
|
||||
$self->testResult(sub {$oStorage->list('/path/to')}, 'file.txt2', 'sub path');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('remove()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oStorage->put($strFile, $strFileContent);
|
||||
$oStorage->put("/path/to/${strFile}2", $strFileContent);
|
||||
$oStorage->put("/path/to/${strFile}3", $strFileContent);
|
||||
$oStorage->put("/path/to/${strFile}4 \@+", $strFileContent);
|
||||
|
||||
$self->testResult(
|
||||
sub {$oStorage->manifest('/')},
|
||||
'{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' .
|
||||
' path/to/file.txt2 => {size => 8, type => f}, path/to/file.txt3 => {size => 8, type => f},' .
|
||||
' path/to/file.txt4 @+ => {size => 8, type => f}}',
|
||||
'check manifest');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oStorage->remove('/path/to', {bRecurse => true})}, true, 'remove subpath');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oStorage->manifest('/')},
|
||||
'{. => {type => d}, file.txt => {size => 8, type => f}}', 'check manifest');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oStorage->remove($strFile)}, true, 'remove file');
|
||||
|
||||
$self->testResult(sub {$oStorage->manifest('/')}, '{. => {type => d}}', 'check manifest');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('info()'))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
storageTest()->put($strFile, $strFileContent);
|
||||
storageTest()->put("${strFile}2", $strFileContent . '2');
|
||||
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev");
|
||||
executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
|
||||
|
||||
$self->testResult(sub {$oStorage->info($strFile)->size()}, 8, 'file size');
|
||||
$self->testResult(sub {$oStorage->info("/path/to/${strFile}2")->size()}, 9, 'file 2 size');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openRead() && S3::FileRead'))
|
||||
{
|
||||
# Create a random 1mb file
|
||||
my $strRandomFile = $self->testPath() . '/random@1mb.bin';
|
||||
executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true});
|
||||
my $strRandom = ${storageTest()->get($strRandomFile)};
|
||||
|
||||
executeTest("$self->{strS3Command} cp ${strRandomFile} s3://pgbackrest-dev/path/to/${strFile}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $tBuffer;
|
||||
my $oFileRead = $self->testResult(sub {$oS3->openRead("/path/to/${strFile}")}, '[object]', 'open read');
|
||||
$self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half');
|
||||
$self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half');
|
||||
$self->testResult(sub {$oFileRead->read(\$tBuffer, 512)}, 0, ' read 0');
|
||||
$self->testResult(length($tBuffer), 1048576, ' check length');
|
||||
$self->testResult(cryptoHashOne('sha1', $tBuffer), cryptoHashOne('sha1', $strRandom), ' check hash');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('openWrite() && S3::FileWrite'))
|
||||
{
|
||||
# Create a random 1mb file
|
||||
my $strRandomFile = $self->testPath() . '/random1mb.bin';
|
||||
executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true});
|
||||
my $strRandom = ${storageTest()->get($strRandomFile)};
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write');
|
||||
$self->testResult(sub {$oFileWrite->name()}, "/path/to/${strFile}", ' check filename');
|
||||
$self->testResult(sub {$oFileWrite->close()}, true, ' close without writing');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}" . '.@')}, '[object]', 'open write');
|
||||
$self->testResult(sub {$oFileWrite->write()}, 0, ' write undef');
|
||||
$self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, ' write');
|
||||
$oFileWrite->close();
|
||||
|
||||
$self->testResult(sub {$oS3->exists("/path/to/${strFile}" . '.@')}, true, 'destination file exists');
|
||||
|
||||
# Test that a premature destroy (from error or otherwise) does not rename the file
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/abort.file" . '.@')}, '[object]', 'open write');
|
||||
$self->testResult(sub {$oFileWrite->write()}, 0, ' write undef');
|
||||
$self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, ' write');
|
||||
|
||||
undef($oFileWrite);
|
||||
$self->testResult(sub {$oS3->exists("/path/to/abort.file")}, false, 'destination file does not exist');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write');
|
||||
|
||||
for (my $iIndex = 1; $iIndex <= 17; $iIndex++)
|
||||
{
|
||||
$self->testResult(sub {$oFileWrite->write(\$strRandom)}, 1024 * 1024, ' write 1mb');
|
||||
}
|
||||
|
||||
$self->testResult(sub {$oFileWrite->close()}, true, ' close');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,184 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# S3 Request Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageS3RequestPerlTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use IO::Socket::SSL;
|
||||
use POSIX qw(strftime);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Http::Client;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Storage::S3::Request;
|
||||
|
||||
use pgBackRestTest::Common::ContainerTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# Port to use for testing
|
||||
####################################################################################################################################
|
||||
use constant HTTPS_TEST_PORT => 9443;
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerResponse
|
||||
####################################################################################################################################
|
||||
sub httpsServerResponse
|
||||
{
|
||||
my $self = shift;
|
||||
my $iResponseCode = shift;
|
||||
my $strContent = shift;
|
||||
|
||||
# Write header
|
||||
$self->{oConnection}->write("HTTP/1.1 ${iResponseCode} GenericMessage\r\n");
|
||||
$self->{oConnection}->write(HTTP_HEADER_CONTENT_LENGTH . ': ' . (defined($strContent) ? length($strContent) : 0) . "\r\n");
|
||||
|
||||
# Write new line before content (even if there isn't any)
|
||||
$self->{oConnection}->write("\r\n");
|
||||
|
||||
# Write content
|
||||
if (defined($strContent))
|
||||
{
|
||||
$self->{oConnection}->write($strContent);
|
||||
}
|
||||
|
||||
# This will block until the connection is closed by the client
|
||||
$self->{oConnection}->read();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerAccept
|
||||
####################################################################################################################################
|
||||
sub httpsServerAccept
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Wait for a connection
|
||||
$self->{oConnection} = $self->{oSocketServer}->accept()
|
||||
or confess "failed to accept or handshake $!, $SSL_ERROR";
|
||||
&log(INFO, " * socket server connected");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServer
|
||||
####################################################################################################################################
|
||||
sub httpsServer
|
||||
{
|
||||
my $self = shift;
|
||||
my $fnServer = shift;
|
||||
|
||||
# Fork off the server
|
||||
if (fork() == 0)
|
||||
{
|
||||
# Run server function
|
||||
$fnServer->();
|
||||
|
||||
exit 0;
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Start the https testing server
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Open the domain socket
|
||||
$self->{oSocketServer} = IO::Socket::SSL->new(
|
||||
LocalAddr => '127.0.0.1', LocalPort => HTTPS_TEST_PORT, Listen => 1, SSL_cert_file => CERT_FAKE_SERVER,
|
||||
SSL_key_file => CERT_FAKE_SERVER_KEY)
|
||||
or confess "unable to open https server for testing: $!";
|
||||
&log(INFO, " * socket server open");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Stop the https testing server
|
||||
####################################################################################################################################
|
||||
sub cleanModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Shutdown server
|
||||
$self->{oSocketServer}->close();
|
||||
&log(INFO, " * socket server closed");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Initialize request object
|
||||
my $oS3Request = new pgBackRest::Storage::S3::Request(
|
||||
BOGUS, BOGUS, BOGUS, BOGUS, BOGUS, {strHost => '127.0.0.1', iPort => HTTPS_TEST_PORT, bVerifySsl => false});
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('success'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request');
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('retry'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request after retries');
|
||||
$self->testException(
|
||||
sub {$oS3Request->request(HTTP_VERB_GET)}, ERROR_PROTOCOL, 'S3 request error after 5 tries \[500\] GenericMessage.*');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -465,6 +465,11 @@ testRun(void)
|
||||
storagePathNP(storageTest, strNew("/path/toot")), AssertError,
|
||||
"absolute path '/path/toot' is not in base path '/path/to'");
|
||||
|
||||
// Path enforcement disabled
|
||||
storagePathEnforceSet(storageTest, false);
|
||||
TEST_RESULT_STR(strPtr(storagePathNP(storageTest, strNew("/bogus"))), "/bogus", "path enforce disabled");
|
||||
storagePathEnforceSet(storageTest, true);
|
||||
|
||||
TEST_ERROR(storagePathNP(storageTest, strNew("<TEST")), AssertError, "end > not found in path expression '<TEST'");
|
||||
TEST_ERROR(
|
||||
storagePathNP(storageTest, strNew("<TEST>" BOGUS_STR)), AssertError,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user