db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-17 01:12:23 +02:00
Code Issues Releases Activity

Add Perl interface to C storage layer.

Browse Source 
Maintaining the storage layer/drivers in two languages is burdensome.  Since the integration tests require the Perl storage layer/drivers we'll need them even after the core code is migrated to C.  Create an interface layer so the Perl code can be removed and new storage drivers/features introduced without adding Perl equivalents.

The goal is to move the integration tests to C so this interface will eventually be removed.  That being the case, the interface was designed for maximum compatibility to ease the transition.  The result looks a bit hacky but we'll improve it as needed until it can be retired.
This commit is contained in:
David Steele
2019-06-26 08:24:58 -04:00
parent bd6c0941e9
commit 4815752ccc
93 changed files with 4412 additions and 12102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
lib/pgBackRest/Protocol/Storage/Helper.pm
View File

@ -13,11 +13,10 @@ use File::Basename qw(basename);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::LibC qw(:storage);
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Remote;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRest::Storage::Local;
####################################################################################################################################
# Storage constants
@ -60,9 +59,8 @@ sub storageDb
{
if (isDbLocal({iRemoteIdx => $iRemoteIdx}))
{
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Local(
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)), new pgBackRest::Storage::Posix::Driver(),
{strTempExtension => STORAGE_TEMP_EXT, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Storage(
STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
else
{
@ -81,54 +79,6 @@ sub storageDb
push @EXPORT, qw(storageDb);
####################################################################################################################################
# storageRepoRule - rules for paths in the repository
####################################################################################################################################
sub storageRepoRule
{
my $strRule = shift;
my $strFile = shift;
my $strStanza = shift;
# Result path and file
my $strResultFile;
# Return archive path
if ($strRule eq STORAGE_REPO_ARCHIVE)
{
$strResultFile = "archive" . (defined($strStanza) ? "/${strStanza}" : '');
# If file is not defined nothing further to do
if (defined($strFile))
{
my ($strArchiveId, $strWalFile) = split('/', $strFile);
# If a WAL file (including .backup)
if (defined($strWalFile) && $strWalFile =~ /^[0-F]{24}/)
{
$strResultFile .= "/${strArchiveId}/" . substr($strWalFile, 0, 16) . "/${strWalFile}";
}
# Else other files
else
{
$strResultFile .= "/${strFile}";
}
}
}
# Return backup path
elsif ($strRule eq STORAGE_REPO_BACKUP)
{
$strResultFile = "backup" . (defined($strStanza) ? "/${strStanza}" : '') . (defined($strFile) ? "/${strFile}" : '');
}
# Else error
else
{
confess &log(ASSERT, "invalid " . STORAGE_REPO . " storage rule ${strRule}");
}
return $strResultFile;
}
####################################################################################################################################
# storageRepo - get repository storage
####################################################################################################################################
@ -146,85 +96,18 @@ sub storageRepo
{name => 'strStanza', optional => true, trace => true},
);
if (!defined($strStanza))
{
if (cfgOptionValid(CFGOPT_STANZA) && cfgOptionTest(CFGOPT_STANZA))
{
$strStanza = cfgOption(CFGOPT_STANZA);
}
else
{
$strStanza = STORAGE_REPO;
}
}
# Create storage if not defined
if (!defined($hStorage->{&STORAGE_REPO}{$strStanza}))
if (!defined($hStorage->{&STORAGE_REPO}))
{
if (isRepoLocal())
{
# Path rules
my $hRule =
{
&STORAGE_REPO_ARCHIVE =>
{
fnRule => \&storageRepoRule,
xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,
},
&STORAGE_REPO_BACKUP =>
{
fnRule => \&storageRepoRule,
xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,
},
};
# Create the driver
my $oDriver;
if (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3))
{
require pgBackRest::Storage::S3::Driver;
$oDriver = new pgBackRest::Storage::S3::Driver(
cfgOption(CFGOPT_REPO_S3_BUCKET), cfgOption(CFGOPT_REPO_S3_ENDPOINT), cfgOption(CFGOPT_REPO_S3_REGION),
cfgOption(CFGOPT_REPO_S3_KEY), cfgOption(CFGOPT_REPO_S3_KEY_SECRET),
{strHost => cfgOption(CFGOPT_REPO_S3_HOST, false), bVerifySsl => cfgOption(CFGOPT_REPO_S3_VERIFY_TLS, false),
strCaPath => cfgOption(CFGOPT_REPO_S3_CA_PATH, false),
strCaFile => cfgOption(CFGOPT_REPO_S3_CA_FILE, false), lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),
strSecurityToken => cfgOption(CFGOPT_REPO_S3_TOKEN, false)});
}
elsif (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_CIFS))
{
require pgBackRest::Storage::Cifs::Driver;
$oDriver = new pgBackRest::Storage::Cifs::Driver();
}
else
{
$oDriver = new pgBackRest::Storage::Posix::Driver();
}
# Set the encryption for the repo
my $strCipherType;
my $strCipherPass;
# If the encryption is not the default (none) then set the user-defined passphrase and magic based on the type
if (cfgOption(CFGOPT_REPO_CIPHER_TYPE) ne CFGOPTVAL_REPO_CIPHER_TYPE_NONE)
{
$strCipherType = cfgOption(CFGOPT_REPO_CIPHER_TYPE);
$strCipherPass = cfgOption(CFGOPT_REPO_CIPHER_PASS);
}
# Create local storage
$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Storage::Local(
cfgOption(CFGOPT_REPO_PATH), $oDriver,
{strTempExtension => STORAGE_TEMP_EXT, hRule => $hRule, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),
strCipherType => $strCipherType, strCipherPassUser => $strCipherPass});
$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(
STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
else
{
# Create remote storage
$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Protocol::Storage::Remote(
$hStorage->{&STORAGE_REPO} = new pgBackRest::Protocol::Storage::Remote(
protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP));
}
}
@ -233,7 +116,7 @@ sub storageRepo
return logDebugReturn
(
$strOperation,
{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}{$strStanza}, trace => true},
{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true},
);
}
@ -249,6 +132,8 @@ sub storageRepoCacheClear
delete($hStorage->{&STORAGE_REPO});
storageRepoFree();
# Return from function and log return values if any
return logDebugReturn($strOperation);
}