mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-03-03 14:52:21 +02:00
ee351682da
"gz" was used as the extension but "gzip" was generally used for function and type naming. With a new compression format on the way, it makes sense to standardize on a single abbreviation to represent a compression format in the code. Since the extension is standard and we must use it, also use the extension for all naming.
361 lines
13 KiB
Perl
361 lines
13 KiB
Perl
####################################################################################################################################
|
|
# Base Storage Module
|
|
####################################################################################################################################
|
|
package pgBackRest::Storage::Base;
|
|
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
use English '-no_match_vars';
|
|
|
|
use Exporter qw(import);
|
|
our @EXPORT = qw();
|
|
use File::Basename qw(dirname);
|
|
|
|
use pgBackRest::Common::Exception;
|
|
use pgBackRest::Common::Io::Base;
|
|
use pgBackRest::Common::Log;
|
|
|
|
####################################################################################################################################
|
|
# Storage constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_LOCAL => '<LOCAL>';
|
|
push @EXPORT, qw(STORAGE_LOCAL);
|
|
|
|
use constant STORAGE_S3 => 's3';
|
|
push @EXPORT, qw(STORAGE_S3);
|
|
use constant STORAGE_POSIX => 'posix';
|
|
push @EXPORT, qw(STORAGE_POSIX);
|
|
|
|
####################################################################################################################################
|
|
# Compress constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_COMPRESS => 'compress';
|
|
push @EXPORT, qw(STORAGE_COMPRESS);
|
|
use constant STORAGE_DECOMPRESS => 'decompress';
|
|
push @EXPORT, qw(STORAGE_DECOMPRESS);
|
|
|
|
####################################################################################################################################
|
|
# Cipher constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_ENCRYPT => 'encrypt';
|
|
push @EXPORT, qw(STORAGE_ENCRYPT);
|
|
use constant STORAGE_DECRYPT => 'decrypt';
|
|
push @EXPORT, qw(STORAGE_DECRYPT);
|
|
use constant CIPHER_MAGIC => 'Salted__';
|
|
push @EXPORT, qw(CIPHER_MAGIC);
|
|
|
|
####################################################################################################################################
|
|
# Filter constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_FILTER_CIPHER_BLOCK => 'pgBackRest::Storage::Filter::CipherBlock';
|
|
push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);
|
|
use constant STORAGE_FILTER_GZ => 'pgBackRest::Storage::Filter::Gz';
|
|
push @EXPORT, qw(STORAGE_FILTER_GZ);
|
|
use constant STORAGE_FILTER_SHA => 'pgBackRest::Storage::Filter::Sha';
|
|
push @EXPORT, qw(STORAGE_FILTER_SHA);
|
|
|
|
####################################################################################################################################
|
|
# Capability constants
|
|
####################################################################################################################################
|
|
# Can the size in the storage be different than what was written? For example, a ZFS filesystem could be doing compression of a
|
|
# backup where compression was not enabled. This affects how repo-size is calculated. If the file system only stores what was
|
|
# written or won't report differently then we can save some time by just setting repo-size to size.
|
|
use constant STORAGE_CAPABILITY_SIZE_DIFF => 'size-diff';
|
|
push @EXPORT, qw(STORAGE_CAPABILITY_SIZE_DIFF);
|
|
|
|
use constant STORAGE_CAPABILITY_LINK => 'link';
|
|
push @EXPORT, qw(STORAGE_CAPABILITY_LINK);
|
|
use constant STORAGE_CAPABILITY_PATH_SYNC => 'path-sync';
|
|
push @EXPORT, qw(STORAGE_CAPABILITY_PATH_SYNC);
|
|
|
|
####################################################################################################################################
|
|
# new
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift;
|
|
|
|
# Create the class hash
|
|
my $self = {};
|
|
bless $self, $class;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
(
|
|
my $strOperation,
|
|
$self->{lBufferMax},
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->new', \@_,
|
|
{name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
|
|
);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
|
|
####################################################################################################################################
|
|
# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of
|
|
# file names.
|
|
####################################################################################################################################
|
|
sub copy
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xSourceFile,
|
|
$xDestinationFile,
|
|
$bSourceOpen,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->copy', \@_,
|
|
{name => 'xSourceFile', required => false},
|
|
{name => 'xDestinationFile'},
|
|
{name => 'bSourceOpen', optional => true, default => false},
|
|
);
|
|
|
|
# Is source/destination an IO object or a file expression?
|
|
my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;
|
|
|
|
# Does the source file exist?
|
|
my $bResult = false;
|
|
|
|
# Copy if the source file exists
|
|
if (defined($oSourceFileIo))
|
|
{
|
|
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);
|
|
|
|
# Use C copy if source and destination are C objects
|
|
if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))
|
|
{
|
|
$bResult = $self->{oStorageC}->copy(
|
|
$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;
|
|
}
|
|
else
|
|
{
|
|
# Open the source file if it is a C object
|
|
$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;
|
|
|
|
if ($bResult)
|
|
{
|
|
# Open the destination file if it is a C object
|
|
if (defined($oDestinationFileIo->{oStorageCWrite}))
|
|
{
|
|
$oDestinationFileIo->open();
|
|
}
|
|
|
|
# Copy the data
|
|
do
|
|
{
|
|
# Read data
|
|
my $tBuffer = '';
|
|
|
|
$oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
|
|
$oDestinationFileIo->write(\$tBuffer);
|
|
}
|
|
while (!$oSourceFileIo->eof());
|
|
|
|
# Close files
|
|
$oSourceFileIo->close();
|
|
$oDestinationFileIo->close();
|
|
}
|
|
}
|
|
}
|
|
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bResult', value => $bResult, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# get - reads a buffer from storage all at once
|
|
####################################################################################################################################
|
|
sub get
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xFile,
|
|
$strCipherPass,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->get', \@_,
|
|
{name => 'xFile', required => false, trace => true},
|
|
{name => 'strCipherPass', optional => true, redact => true},
|
|
);
|
|
|
|
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
|
|
# if the repo has a user passphrase defined - else pass undef
|
|
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead(
|
|
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()})) : undef;
|
|
|
|
# Read only if there is something to read from
|
|
my $tContent;
|
|
my $lSize = 0;
|
|
|
|
if (defined($oFileIo))
|
|
{
|
|
my $lSizeRead;
|
|
|
|
do
|
|
{
|
|
$lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
|
|
$lSize += $lSizeRead;
|
|
}
|
|
while ($lSizeRead != 0);
|
|
|
|
# Close the file
|
|
$oFileIo->close();
|
|
|
|
# If nothing was read then set to undef
|
|
if ($lSize == 0)
|
|
{
|
|
$tContent = undef;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathAbsolute - generate an absolute path from an absolute base path and a relative path
|
|
####################################################################################################################################
|
|
sub pathAbsolute
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strBasePath,
|
|
$strPath
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '::pathAbsolute', \@_,
|
|
{name => 'strBasePath', trace => true},
|
|
{name => 'strPath', trace => true}
|
|
);
|
|
|
|
# Working variables
|
|
my $strAbsolutePath;
|
|
|
|
# If the path is already absolute
|
|
if (index($strPath, '/') == 0)
|
|
{
|
|
$strAbsolutePath = $strPath;
|
|
}
|
|
# Else make it absolute using the base path
|
|
else
|
|
{
|
|
# Make sure the absolute path is really absolute
|
|
if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)
|
|
{
|
|
confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE);
|
|
}
|
|
|
|
while (index($strPath, '..') == 0)
|
|
{
|
|
$strBasePath = dirname($strBasePath);
|
|
$strPath = substr($strPath, 2);
|
|
|
|
if (index($strPath, '/') == 0)
|
|
{
|
|
$strPath = substr($strPath, 1);
|
|
}
|
|
}
|
|
|
|
$strAbsolutePath = "${strBasePath}/${strPath}";
|
|
}
|
|
|
|
# Make sure the result is really an absolute path
|
|
if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)
|
|
{
|
|
confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# put - writes a buffer out to storage all at once
|
|
####################################################################################################################################
|
|
sub put
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xFile,
|
|
$xContent,
|
|
$strCipherPass,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->put', \@_,
|
|
{name => 'xFile', trace => true},
|
|
{name => 'xContent', required => false, trace => true},
|
|
{name => 'strCipherPass', optional => true, trace => true, redact => true},
|
|
);
|
|
|
|
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
|
|
# the repo has a user passphrase defined - else pass undef
|
|
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite(
|
|
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()});
|
|
|
|
# Determine size of content
|
|
my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
|
|
|
|
# Write only if there is something to write
|
|
if ($lSize > 0)
|
|
{
|
|
$oFileIo->write(ref($xContent) ? $xContent : \$xContent);
|
|
}
|
|
# Else open the file so a zero length file is created (since file is not opened until first write)
|
|
else
|
|
{
|
|
$oFileIo->open();
|
|
}
|
|
|
|
# Close the file
|
|
$oFileIo->close();
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'lSize', value => $lSize, trace => true},
|
|
);
|
|
}
|
|
|
|
1;
|