mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-14 10:13:05 +02:00
348278bb68
Scanning the entire backup directory can be very expensive if there are a lot of small tables. The backup manifest contains the backup directory list so use it to perform syncs instead of scanning the backup directory.
318 lines
10 KiB
Perl
318 lines
10 KiB
Perl
####################################################################################################################################
|
|
# Base Storage Module
|
|
####################################################################################################################################
|
|
package pgBackRest::Storage::Base;
|
|
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
use English '-no_match_vars';
|
|
|
|
use Exporter qw(import);
|
|
our @EXPORT = qw();
|
|
use File::Basename qw(dirname);
|
|
|
|
use pgBackRest::Common::Exception;
|
|
use pgBackRest::Common::Io::Base;
|
|
use pgBackRest::Common::Log;
|
|
|
|
####################################################################################################################################
|
|
# Compress constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_COMPRESS => 'compress';
|
|
push @EXPORT, qw(STORAGE_COMPRESS);
|
|
use constant STORAGE_DECOMPRESS => 'decompress';
|
|
push @EXPORT, qw(STORAGE_DECOMPRESS);
|
|
|
|
####################################################################################################################################
|
|
# Cipher constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_ENCRYPT => 'encrypt';
|
|
push @EXPORT, qw(STORAGE_ENCRYPT);
|
|
use constant STORAGE_DECRYPT => 'decrypt';
|
|
push @EXPORT, qw(STORAGE_DECRYPT);
|
|
use constant CIPHER_MAGIC => 'Salted__';
|
|
push @EXPORT, qw(CIPHER_MAGIC);
|
|
|
|
####################################################################################################################################
|
|
# Capability constants
|
|
####################################################################################################################################
|
|
use constant STORAGE_CAPABILITY_LINK => 'link';
|
|
push @EXPORT, qw(STORAGE_CAPABILITY_LINK);
|
|
use constant STORAGE_CAPABILITY_PATH_SYNC => 'path-sync';
|
|
push @EXPORT, qw(STORAGE_CAPABILITY_PATH_SYNC);
|
|
|
|
####################################################################################################################################
|
|
# new
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift;
|
|
|
|
# Create the class hash
|
|
my $self = {};
|
|
bless $self, $class;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
(
|
|
my $strOperation,
|
|
$self->{lBufferMax},
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->new', \@_,
|
|
{name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
|
|
);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# copy - copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed
|
|
# instead of file names.
|
|
####################################################################################################################################
|
|
sub copy
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xSourceFile,
|
|
$xDestinationFile,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->copy', \@_,
|
|
{name => 'xSourceFile', required => false},
|
|
{name => 'xDestinationFile', required => false},
|
|
);
|
|
|
|
# Was the file copied?
|
|
my $bResult = false;
|
|
|
|
# Is source an IO object or a file expression?
|
|
my $oSourceFileIo =
|
|
defined($xSourceFile) ?
|
|
(ref($xSourceFile) ? $xSourceFile : $self->openRead($self->pathGet($xSourceFile))) : undef;
|
|
|
|
# Proceed if source file exists
|
|
if (defined($oSourceFileIo))
|
|
{
|
|
# Is destination an IO object or a file expression?
|
|
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($self->pathGet($xDestinationFile));
|
|
|
|
# Copy the data
|
|
my $lSizeRead;
|
|
|
|
do
|
|
{
|
|
# Read data
|
|
my $tBuffer = '';
|
|
|
|
$lSizeRead = $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
|
|
$oDestinationFileIo->write(\$tBuffer);
|
|
}
|
|
while ($lSizeRead != 0);
|
|
|
|
# Close files
|
|
$oSourceFileIo->close();
|
|
$oDestinationFileIo->close();
|
|
|
|
# File was copied
|
|
$bResult = true;
|
|
}
|
|
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bResult', value => $bResult, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# get - reads a buffer from storage all at once
|
|
####################################################################################################################################
|
|
sub get
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xFile,
|
|
$strCipherPass,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->get', \@_,
|
|
{name => 'xFile', required => false, trace => true},
|
|
{name => 'strCipherPass', optional => true, redact => true},
|
|
);
|
|
|
|
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
|
|
# if the repo has a user passphrase defined - else pass undef
|
|
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead(
|
|
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()})) : undef;
|
|
|
|
# Read only if there is something to read from
|
|
my $tContent;
|
|
my $lSize = 0;
|
|
|
|
if (defined($oFileIo))
|
|
{
|
|
my $lSizeRead;
|
|
|
|
do
|
|
{
|
|
$lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
|
|
$lSize += $lSizeRead;
|
|
}
|
|
while ($lSizeRead != 0);
|
|
|
|
# Close the file
|
|
$oFileIo->close();
|
|
|
|
# If nothing was read then set to undef
|
|
if ($lSize == 0)
|
|
{
|
|
$tContent = undef;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# pathAbsolute - generate an absolute path from an absolute base path and a relative path
|
|
####################################################################################################################################
|
|
sub pathAbsolute
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strBasePath,
|
|
$strPath
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '::pathAbsolute', \@_,
|
|
{name => 'strBasePath', trace => true},
|
|
{name => 'strPath', trace => true}
|
|
);
|
|
|
|
# Working variables
|
|
my $strAbsolutePath;
|
|
|
|
# If the path is already absolute
|
|
if (index($strPath, '/') == 0)
|
|
{
|
|
$strAbsolutePath = $strPath;
|
|
}
|
|
# Else make it absolute using the base path
|
|
else
|
|
{
|
|
# Make sure the absolute path is really absolute
|
|
if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)
|
|
{
|
|
confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE);
|
|
}
|
|
|
|
while (index($strPath, '..') == 0)
|
|
{
|
|
$strBasePath = dirname($strBasePath);
|
|
$strPath = substr($strPath, 2);
|
|
|
|
if (index($strPath, '/') == 0)
|
|
{
|
|
$strPath = substr($strPath, 1);
|
|
}
|
|
}
|
|
|
|
$strAbsolutePath = "${strBasePath}/${strPath}";
|
|
}
|
|
|
|
# Make sure the result is really an absolute path
|
|
if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)
|
|
{
|
|
confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# put - writes a buffer out to storage all at once
|
|
####################################################################################################################################
|
|
sub put
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$xFile,
|
|
$xContent,
|
|
$strCipherPass,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->put', \@_,
|
|
{name => 'xFile', trace => true},
|
|
{name => 'xContent', required => false, trace => true},
|
|
{name => 'strCipherPass', optional => true, trace => true, redact => true},
|
|
);
|
|
|
|
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
|
|
# the repo has a user passphrase defined - else pass undef
|
|
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite(
|
|
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()});
|
|
|
|
# Determine size of content
|
|
my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
|
|
|
|
# Write only if there is something to write
|
|
if ($lSize > 0)
|
|
{
|
|
$oFileIo->write(ref($xContent) ? $xContent : \$xContent);
|
|
}
|
|
# Else open the file so a zero length file is created (since file is not opened until first write)
|
|
else
|
|
{
|
|
$oFileIo->open();
|
|
}
|
|
|
|
# Close the file
|
|
$oFileIo->close();
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'lSize', value => $lSize, trace => true},
|
|
);
|
|
}
|
|
|
|
1;
|