1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
pgbackrest/lib/pgBackRest/Storage/Local.pm
David Steele 7cf955425e The C library is now required.
This eliminates conditional loading and eases development of new library features.
2017-11-26 17:45:00 -05:00

921 lines
29 KiB
Perl

####################################################################################################################################
# Local Storage
#
# Implements storage functionality using drivers.
####################################################################################################################################
package pgBackRest::Storage::Local;
use parent 'pgBackRest::Storage::Base';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Sha;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathBase,
$oDriver,
$hRule,
$bAllowTemp,
$strTempExtension,
$strDefaultPathMode,
$strDefaultFileMode,
$lBufferMax,
$strCipherType,
$strCipherPassUser,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strPathBase'},
{name => 'oDriver'},
{name => 'hRule', optional => true},
{name => 'bAllowTemp', optional => true, default => true},
{name => 'strTempExtension', optional => true, default => 'tmp'},
{name => 'strDefaultPathMode', optional => true, default => '0750'},
{name => 'strDefaultFileMode', optional => true, default => '0640'},
{name => 'lBufferMax', optional => true},
{name => 'strCipherType', optional => true},
{name => 'strCipherPassUser', optional => true, redact => true},
);
# Create class
my $self = $class->SUPER::new({lBufferMax => $lBufferMax});
bless $self, $class;
$self->{strPathBase} = $strPathBase;
$self->{oDriver} = $oDriver;
$self->{hRule} = $hRule;
$self->{bAllowTemp} = $bAllowTemp;
$self->{strTempExtension} = $strTempExtension;
$self->{strDefaultPathMode} = $strDefaultPathMode;
$self->{strDefaultFileMode} = $strDefaultFileMode;
$self->{strCipherType} = $strCipherType;
$self->{strCipherPassUser} = $strCipherPassUser;
if (defined($self->{strCipherType}))
{
require pgBackRest::Storage::Filter::CipherBlock;
pgBackRest::Storage::Filter::CipherBlock->import();
}
# Set temp extension in driver
$self->driver()->tempExtensionSet($self->{strTempExtension}) if $self->driver()->can('tempExtensionSet');
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# exists - check if file exists
####################################################################################################################################
sub exists
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileExp,
) =
logDebugParam
(
__PACKAGE__ . '->exists', \@_,
{name => 'strFileExp'},
);
# Check exists
my $bExists = $self->driver()->exists($self->pathGet($strFileExp));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $bExists}
);
}
####################################################################################################################################
# hashSize - calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from
# openRead/openWrite must be passed instead of file names.
####################################################################################################################################
sub hashSize
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
) =
logDebugParam
(
__PACKAGE__ . '->hashSize', \@_,
{name => 'xFileExp'},
);
# Set operation variables
my $strHash;
my $lSize;
# Is this an IO object or a file expression?
my $oFileIo = defined($xFileExp) ? (ref($xFileExp) ? $xFileExp : $self->openRead($self->pathGet($xFileExp))) : undef;
if (defined($oFileIo))
{
$lSize = 0;
my $oShaIo = new pgBackRest::Storage::Filter::Sha($oFileIo);
my $lSizeRead;
do
{
my $tContent;
$lSizeRead = $oShaIo->read(\$tContent, $self->{lBufferMax});
$lSize += $lSizeRead;
}
while ($lSizeRead != 0);
# Close the file
$oShaIo->close();
# Get the hash
$strHash = $oShaIo->result(STORAGE_FILTER_SHA);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strHash', value => $strHash},
{name => 'lSize', value => $lSize}
);
}
####################################################################################################################################
# info - get information for path/file
####################################################################################################################################
sub info
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathFileExp,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '::fileStat', \@_,
{name => 'strPathFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Stat the path/file
my $oInfo = $self->driver()->info($self->pathGet($strPathFileExp), {bIgnoreMissing => $bIgnoreMissing});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oInfo', value => $oInfo, trace => true}
);
}
####################################################################################################################################
# linkCreate - create a link
####################################################################################################################################
sub linkCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourcePathFileExp,
$strDestinationLinkExp,
$bHard,
$bRelative,
$bPathCreate,
$bIgnoreExists,
) =
logDebugParam
(
__PACKAGE__ . '->linkCreate', \@_,
{name => 'strSourcePathFileExp'},
{name => 'strDestinationLinkExp'},
{name => 'bHard', optional=> true, default => false},
{name => 'bRelative', optional=> true, default => false},
{name => 'bPathCreate', optional=> true, default => true},
{name => 'bIgnoreExists', optional => true, default => false},
);
# Get source and destination paths
my $strSourcePathFile = $self->pathGet($strSourcePathFileExp);
my $strDestinationLink = $self->pathGet($strDestinationLinkExp);
# Generate relative path if requested
if ($bRelative)
{
# Determine how much of the paths are common
my @strySource = split('/', $strSourcePathFile);
my @stryDestination = split('/', $strDestinationLink);
while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0])
{
shift(@strySource);
shift(@stryDestination);
}
# Add relative path sections
$strSourcePathFile = '';
for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++)
{
$strSourcePathFile .= '../';
}
# Add path to source
$strSourcePathFile .= join('/', @strySource);
logDebugMisc
(
$strOperation, 'apply relative path',
{name => 'strSourcePathFile', value => $strSourcePathFile, trace => true}
);
}
# Create the link
$self->driver()->linkCreate(
$strSourcePathFile, $strDestinationLink, {bHard => $bHard, bPathCreate => $bPathCreate, bIgnoreExists => $bIgnoreExists});
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# list - list all files/paths in path
####################################################################################################################################
sub list
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$strExpression,
$strSortOrder,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->list', \@_,
{name => 'strPathExp'},
{name => 'strExpression', optional => true},
{name => 'strSortOrder', optional => true, default => 'forward'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Get file list
my $rstryFileList = $self->driver()->list(
$self->pathGet($strPathExp), {strExpression => $strExpression, bIgnoreMissing => $bIgnoreMissing});
# Apply expression if defined
if (defined($strExpression))
{
@{$rstryFileList} = grep(/$strExpression/i, @{$rstryFileList});
}
# Reverse sort
if ($strSortOrder eq 'reverse')
{
@{$rstryFileList} = sort {$b cmp $a} @{$rstryFileList};
}
# Normal sort
else
{
@{$rstryFileList} = sort @{$rstryFileList};
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryFileList', value => $rstryFileList}
);
}
####################################################################################################################################
# manifest - build path/file/link manifest starting with base path and including all subpaths
####################################################################################################################################
sub manifest
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->manifest', \@_,
{name => 'strPathExp'},
);
my $hManifest = $self->driver()->manifest($self->pathGet($strPathExp));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hManifest', value => $hManifest, trace => true}
);
}
####################################################################################################################################
# move - move path/file
####################################################################################################################################
sub move
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourcePathFileExp,
$strDestinationPathFileExp,
$bPathCreate,
) =
logDebugParam
(
__PACKAGE__ . '->move', \@_,
{name => 'strSourcePathExp'},
{name => 'strDestinationPathExp'},
{name => 'bPathCreate', optional => true, default => false, trace => true},
);
# Set operation variables
$self->driver()->move(
$self->pathGet($strSourcePathFileExp), $self->pathGet($strDestinationPathFileExp), {bPathCreate => $bPathCreate});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# openRead - open file for reading
####################################################################################################################################
sub openRead
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
$bIgnoreMissing,
$rhyFilter,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->openRead', \@_,
{name => 'xFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
{name => 'rhyFilter', optional => true},
{name => 'strCipherPass', optional => true, redact => true},
);
# Open the file
my $oFileIo = $self->driver()->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing});
# Apply filters if file is defined
if (defined($oFileIo))
{
# If cipher is set then add the filter so that decryption is the first filter applied to the data read before any of the
# other filters
if (defined($self->cipherType()))
{
$oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new(
$oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser(),
{strMode => STORAGE_DECRYPT});
}
# Apply any other filters
if (defined($rhyFilter))
{
foreach my $rhFilter (@{$rhyFilter})
{
$oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => $oFileIo, trace => true},
);
}
####################################################################################################################################
# openWrite - open file for writing
####################################################################################################################################
sub openWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
$strMode,
$strUser,
$strGroup,
$lTimestamp,
$bAtomic,
$bPathCreate,
$rhyFilter,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->openWrite', \@_,
{name => 'xFileExp'},
{name => 'strMode', optional => true, default => $self->{strDefaultFileMode}},
{name => 'strUser', optional => true},
{name => 'strGroup', optional => true},
{name => 'lTimestamp', optional => true},
{name => 'bAtomic', optional => true, default => false},
{name => 'bPathCreate', optional => true, default => false},
{name => 'rhyFilter', optional => true},
{name => 'strCipherPass', optional => true, redact => true},
);
# Open the file
my $oFileIo = $self->driver()->openWrite($self->pathGet($xFileExp),
{strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,
bAtomic => $bAtomic});
# If cipher is set then add filter so that encryption is performed just before the data is actually written
if (defined($self->cipherType()))
{
$oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new(
$oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser());
}
# Apply any other filters
if (defined($rhyFilter))
{
foreach my $rhFilter (reverse(@{$rhyFilter}))
{
$oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => $oFileIo, trace => true},
);
}
####################################################################################################################################
# owner - change ownership of path/file
####################################################################################################################################
sub owner
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathFileExp,
$strUser,
$strGroup
) =
logDebugParam
(
__PACKAGE__ . '->owner', \@_,
{name => 'strPathFileExp'},
{name => 'strUser', required => false},
{name => 'strGroup', required => false}
);
# Set ownership
$self->driver()->owner($self->pathGet($strPathFileExp), {strUser => $strUser, strGroup => $strGroup});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# pathCreate - create path
####################################################################################################################################
sub pathCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$strMode,
$bIgnoreExists,
$bCreateParent,
) =
logDebugParam
(
__PACKAGE__ . '->pathCreate', \@_,
{name => 'strPathExp'},
{name => 'strMode', optional => true, default => $self->{strDefaultPathMode}},
{name => 'bIgnoreExists', optional => true, default => false},
{name => 'bCreateParent', optional => true, default => false},
);
# Create path
$self->driver()->pathCreate(
$self->pathGet($strPathExp), {strMode => $strMode, bIgnoreExists => $bIgnoreExists, bCreateParent => $bCreateParent});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# pathExists - check if path exists
####################################################################################################################################
sub pathExists
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->pathExists', \@_,
{name => 'strPathExp'},
);
# Check exists
my $bExists = $self->driver()->pathExists($self->pathGet($strPathExp));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $bExists}
);
}
####################################################################################################################################
# pathGet - resolve a path expression into an absolute path
####################################################################################################################################
sub pathGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp, # File that that needs to be translated to a path
$bTemp, # Return the temp file name
) =
logDebugParam
(
__PACKAGE__ . '->pathGet', \@_,
{name => 'strPathExp', required => false, trace => true},
{name => 'bTemp', optional => true, default => false, trace => true},
);
# Path and file to be returned
my $strPath;
my $strFile;
# Is this an absolute path type?
my $bAbsolute = false;
if (defined($strPathExp) && index($strPathExp, qw(/)) == 0)
{
$bAbsolute = true;
$strPath = $strPathExp;
}
else
{
# Is it a rule type
if (defined($strPathExp) && index($strPathExp, qw(<)) == 0)
{
# Extract the rule type
my $iPos = index($strPathExp, qw(>));
if ($iPos == -1)
{
confess &log(ASSERT, "found < but not > in '${strPathExp}'");
}
my $strType = substr($strPathExp, 0, $iPos + 1);
# Extract the filename
if ($iPos < length($strPathExp) - 1)
{
$strFile = substr($strPathExp, $iPos + 2);
}
# Lookup the rule
if (!defined($self->{hRule}->{$strType}))
{
confess &log(ASSERT, "storage rule '${strType}' does not exist");
}
# If rule is a ref then call the function
if (ref($self->{hRule}->{$strType}))
{
$strPath = $self->pathBase();
$strFile = $self->{hRule}{$strType}{fnRule}->($strType, $strFile, $self->{hRule}{$strType}{xData});
}
# Else get the path
else
{
$strPath = $self->pathBase() . ($self->pathBase() =~ /\/$/ ? '' : qw{/}) . $self->{hRule}->{$strType};
}
}
# Else it must be relative
else
{
$strPath = $self->pathBase();
$strFile = $strPathExp;
}
}
# Make sure a temp file is valid for this type and file
if ($bTemp)
{
# Error when temp files are not allowed
if (!$self->{bAllowTemp})
{
confess &log(ASSERT, "temp file not supported for storage '" . $self->pathBase() . "'");
}
# The file must be defined
if (!$bAbsolute)
{
if (!defined($strFile))
{
confess &log(ASSERT, 'file part must be defined when temp file specified');
}
}
}
# Combine path and file
$strPath .= defined($strFile) ? ($strPath =~ /\/$/ ? '' : qw{/}) . "${strFile}" : '';
# Add temp extension
$strPath .= $bTemp ? ".$self->{strTempExtension}" : '';
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strPath', value => $strPath, trace => true}
);
}
####################################################################################################################################
# pathSync - perform driver sync operation on path
####################################################################################################################################
sub pathSync
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->pathSync', \@_,
{name => 'strPathExp'},
{name => 'bRecurse', default => false},
);
$self->driver()->pathSync($self->pathGet($strPathExp), {bRecurse => true});
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# remove - remove path/file
####################################################################################################################################
sub remove
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xstryPathFileExp,
$bIgnoreMissing,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->remove', \@_,
{name => 'xstryPathFileExp'},
{name => 'bIgnoreMissing', optional => true, default => true},
{name => 'bRecurse', optional => true, default => false, trace => true},
);
# Evaluate expressions for all files
my @stryPathFileExp;
if (ref($xstryPathFileExp))
{
foreach my $strPathFileExp (@{$xstryPathFileExp})
{
push(@stryPathFileExp, $self->pathGet($strPathFileExp));
}
}
# Remove path(s)/file(s)
my $bRemoved = $self->driver()->remove(
ref($xstryPathFileExp) ? \@stryPathFileExp : $self->pathGet($xstryPathFileExp),
{bIgnoreMissing => $bIgnoreMissing, bRecurse => $bRecurse});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bRemoved', value => $bRemoved}
);
}
####################################################################################################################################
# encrypted - determine if the file is encrypted or not
####################################################################################################################################
sub encrypted
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileName,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->encrypted', \@_,
{name => 'strFileName'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
my $tMagicSignature;
my $bEncrypted = false;
# Open the file via the driver
my $oFile = $self->driver()->openRead($self->pathGet($strFileName), {bIgnoreMissing => $bIgnoreMissing});
# If the file does not exist because we're ignoring missing (else it would error before this is executed) then determine if it
# should be encrypted based on the repo
if (!defined($oFile))
{
if (defined($self->{strCipherType}))
{
$bEncrypted = true;
}
}
else
{
# If the file does exist, then read the magic signature
my $lSizeRead = $oFile->read(\$tMagicSignature, length(CIPHER_MAGIC));
# Close the file handle
$oFile->close();
# If the file is able to be read, then if it is encrypted it must at least have the magic signature, even if it were
# originally a 0 byte file
if (($lSizeRead > 0) && substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC)
{
$bEncrypted = true;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bEncrypted', value => $bEncrypted}
);
}
####################################################################################################################################
# encryptionValid - determine if encyption set properly based on the value passed
####################################################################################################################################
sub encryptionValid
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bEncrypted,
) =
logDebugParam
(
__PACKAGE__ . '->encryptionValid', \@_,
{name => 'bEncrypted'},
);
my $bValid = true;
# If encryption is set on the file then make sure the repo is encrypted and visa-versa
if ($bEncrypted)
{
if (!defined($self->{strCipherType}))
{
$bValid = false;
}
}
else
{
if (defined($self->{strCipherType}))
{
$bValid = false;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bValid', value => $bValid}
);
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub pathBase {shift->{strPathBase}}
sub driver {shift->{oDriver}}
sub cipherType {shift->{strCipherType}}
sub cipherPassUser {shift->{strCipherPassUser}}
1;