db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Remove obsolete methods in pgBackRest::Storage::Storage module.

Browse Source 
All the methods in this module will need to be implemented via the command-line in order to get rid of LibC, so the first step is to reduce the code in the module as much as possible.

First remove storageDb() and use storageTest() instead.  Then create storageTest() using pgBackRestTest::Common::Storage which has no dependencies on LibC.  Now the only storage using the LibC interface is storageRepo().

Remove all link functions since those operations cannot be performed on a repo unless it is Posix, in which case the LibC interface is not needed.  Same for owner().

Remove pathSync() because syncs are not required in the tests.  No test data is reused after a crash.

Path create/exists functions should never be explicitly performed on a repo so remove those.  File exists can be implemented by calling info() instead.

Remove encryption detection functions which were only used by Backup/Archive::Info reconstruct() which are now obsolete.

Remove all filters except pgBackRest::Storage::Filter::CipherBlock since they are not being used.  That also means there are no filters returning results so remove all the result code.

Move hashSize() and pathAbsolute() into pgBackRest::Storage::Base where they can be shared between pgBackRest::Storage::Storage and pgBackRestTest::Common::Storage.
This commit is contained in:
David Steele 2020-03-06 14:10:09 -05:00
parent 00647c7109
commit 02aa03d1a2
21 changed files with 113 additions and 1273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
lib/pgBackRest/Archive/Info.pm
View File

@ -334,149 +334,6 @@ sub create
return logDebugReturn($strOperation);
}
####################################################################################################################################
# reconstruct
#
# Reconstruct the info file from the existing directory and files
####################################################################################################################################
sub reconstruct
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCurrentDbVersion,
$ullCurrentDbSysId,
) =
logDebugParam
(
__PACKAGE__ . '->reconstruct', \@_,
{name => 'strCurrentDbVersion'},
{name => 'ullCurrentDbSysId'},
);
my $strInvalidFileStructure = undef;
my @stryArchiveId = storageRepo()->list(
$self->{strArchiveClusterPath}, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});
my %hDbHistoryVersion;
# Get the db-version and db-id (history id) from the upper level directory names, e.g. 9.4-1
foreach my $strArchiveId (@stryArchiveId)
{
my ($strDbVersion, $iDbHistoryId) = split("-", $strArchiveId);
$hDbHistoryVersion{$iDbHistoryId} = $strDbVersion;
}
# Loop through the DBs in the order they were created as indicated by the db-id so that the last one is set in the db section
foreach my $iDbHistoryId (sort {$a <=> $b} keys %hDbHistoryVersion)
{
my $strDbVersion = $hDbHistoryVersion{$iDbHistoryId};
my $strVersionDir = $strDbVersion . "-" . $iDbHistoryId;
# Get the name of the first archive directory
my $strArchiveDir = (storageRepo()->list(
$self->{strArchiveClusterPath} . "/${strVersionDir}",
{strExpression => REGEX_ARCHIVE_DIR_WAL, bIgnoreMissing => true}))[0];
# Continue if any file structure or missing files info
if (!defined($strArchiveDir))
{
$strInvalidFileStructure = "found empty directory " . $self->{strArchiveClusterPath} . "/${strVersionDir}";
next;
}
# ??? Should probably make a function in ArchiveCommon
my $strArchiveFile = (storageRepo()->list(
$self->{strArchiveClusterPath} . "/${strVersionDir}/${strArchiveDir}",
{strExpression => "^[0-F]{24}(\\.partial){0,1}(-[0-f]+){0,1}(\\." . COMPRESS_EXT . "){0,1}\$",
bIgnoreMissing => true}))[0];
# Continue if any file structure or missing files info
if (!defined($strArchiveFile))
{
$strInvalidFileStructure =
"found empty directory " . $self->{strArchiveClusterPath} . "/${strVersionDir}/${strArchiveDir}";
next;
}
# Get the full path for the file
my $strArchiveFilePath = $self->{strArchiveClusterPath}."/${strVersionDir}/${strArchiveDir}/${strArchiveFile}";
# Get the db-system-id from the WAL file depending on the version of postgres
my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;
# Error if the file encryption setting is not valid for the repo
if (!storageRepo()->encryptionValid(storageRepo()->encrypted($strArchiveFilePath)))
{
confess &log(ERROR, "encryption incompatible for '$strArchiveFilePath'" .
"\nHINT: is or was the repo encrypted?", ERROR_CRYPTO);
}
# If the file is encrypted, then the passphrase from the info file is required, else getEncryptionKeySub returns undefined
my $oFileIo = storageRepo()->openRead(
$strArchiveFilePath,
{rhyFilter => $strArchiveFile =~ ('\.' . COMPRESS_EXT . '$') ?
[{strClass => STORAGE_FILTER_GZ, rxyParam => [STORAGE_DECOMPRESS, false]}] : undef,
strCipherPass => $self->cipherPassSub()});
$oFileIo->open();
my $tBlock;
$oFileIo->read(\$tBlock, 512);
$oFileIo->close();
# Get the required data from the file that was pulled into scalar $tBlock
my ($iMagic, $iFlag, $junk, $ullDbSysId) = unpack('SSa' . $iSysIdOffset . 'Q', $tBlock);
if (!defined($ullDbSysId))
{
confess &log(ERROR, "unable to read database system identifier", ERROR_FILE_READ);
}
# Fill db section and db history section
$self->dbSectionSet($strDbVersion, $ullDbSysId, $iDbHistoryId);
}
# If the DB section does not exist, then there were no valid directories to read from so create the DB and History sections.
if (!$self->test(INFO_ARCHIVE_SECTION_DB))
{
$self->create($strCurrentDbVersion, $ullCurrentDbSysId, false);
}
# Else if it does exist but does not match the current DB, then update the DB section
else
{
# Turn off console logging to control when to display the error
logDisable();
eval
{
$self->check($strCurrentDbVersion, $ullCurrentDbSysId, false);
logEnable();
return true;
}
or do
{
# Reset the console logging
logEnable();
# Confess unhandled errors
confess $EVAL_ERROR if (exceptionCode($EVAL_ERROR) != ERROR_ARCHIVE_MISMATCH);
# Update the DB section if it does not match the current database
$self->dbSectionSet($strCurrentDbVersion, $ullCurrentDbSysId, $self->dbHistoryIdGet(false)+1);
};
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strInvalidFileStructure', value => $strInvalidFileStructure}
);
}
####################################################################################################################################
# dbHistoryIdGet
#

166
lib/pgBackRest/Backup/Info.pm
View File

@ -111,7 +111,6 @@ sub new
(
$strOperation,
$strBackupClusterPath,
$bValidate,
$bRequired,
$oStorage,
$bLoad, # Should the file attemp to be loaded?
@ -122,7 +121,6 @@ sub new
(
__PACKAGE__ . '->new', \@_,
{name => 'strBackupClusterPath'},
{name => 'bValidate', default => true},
{name => 'bRequired', default => true},
{name => 'oStorage', optional => true, default => storageRepo()},
{name => 'bLoad', optional => true, default => true},
@ -175,12 +173,6 @@ sub new
$self->{strBackupClusterPath} = $strBackupClusterPath;
$self->{oStorage} = $oStorage;
# Validate the backup info
if ($bValidate)
{
$self->validate();
}
# Return from function and log return values if any
return logDebugReturn
(
@ -189,164 +181,6 @@ sub new
);
}
####################################################################################################################################
# validate
#
# Confirm the file exists and reconstruct as necessary.
####################################################################################################################################
sub validate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->validate');
# Confirm the info file exists with the DB section
$self->confirmExists();
$self->reconstruct();
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# reconstruct
#
# Compare the backup info against the actual backups in the repository. Reconstruct the file based on manifests missing or no
# longer valid.
####################################################################################################################################
sub reconstruct
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bSave,
$bRequired, # If false then must be creating or reconstructing so the DB info must be supplied
$strDbVersion,
$ullDbSysId,
$iControlVersion,
$iCatalogVersion,
) =
logDebugParam
(
__PACKAGE__ . '->reconstruct', \@_,
{name => 'bSave', default => true},
{name => 'bRequired', default => true},
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
{name => 'iControlVersion', required => false},
{name => 'iCatalogVersion', required => false},
);
# Check for backups that are not in FILE_BACKUP_INFO
foreach my $strBackup ($self->{oStorage}->list(
$self->{strBackupClusterPath}, {strExpression => backupRegExpGet(true, true, true)}))
{
my $strManifestFile = "$self->{strBackupClusterPath}/${strBackup}/" . FILE_MANIFEST;
# ??? Check for and move history files that were not moved before and maybe don't consider it to be an error when they
# can't be moved. This would also be true for the first move attempt in Backup->process();
if (!$self->current($strBackup) && $self->{oStorage}->exists($strManifestFile))
{
my $oManifest = pgBackRest::Manifest->new($strManifestFile,
{strCipherPass => ($self->{oStorage}->encrypted($strManifestFile)) ? $self->cipherPassSub() : undef});
# If we are reconstructing, then we need to be sure this db-id and version is in the history section. Also if it
# has a db-id greater than anything in the history section, then add it to the db section.
if (!$bRequired)
{
my $hDbList = $self->dbHistoryList();
my $iDbId = $oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID);
my $iDbIdMax = 0;
my $ullDbSysId = $oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_SYSTEM_ID);
my $strDbVersion = $oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION);
# If this is the max history id then set the db section
foreach my $iDbHistoryId (keys %{$hDbList})
{
# If the current history ID is greater than the running max, then set it to the current id
if ($iDbHistoryId > $iDbIdMax)
{
$iDbIdMax = $iDbHistoryId;
}
}
if ($iDbId >= $iDbIdMax)
{
$self->dbSectionSet($strDbVersion, $oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CONTROL),
$oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG), $ullDbSysId, $iDbId);
}
}
&log(WARN, "backup ${strBackup} found in repository added to " . FILE_BACKUP_INFO);
$self->add($oManifest, $bSave, $bRequired);
}
}
# If reconstructing, make sure the DB section is correct
if (!$bRequired)
{
# If any database info is missing, then assert
if (!defined($strDbVersion) || !defined($ullDbSysId) || !defined($iControlVersion) || !defined($iCatalogVersion))
{
confess &log(ASSERT, "backup info cannot be reconstructed without database information");
}
# If the DB section does not exist then create the db and history section
elsif (!$self->test(INFO_BACKUP_SECTION_DB))
{
$self->create($strDbVersion, $ullDbSysId, $iControlVersion, $iCatalogVersion, $bSave);
}
# Else update the DB section if it does not match the current database
else
{
# Turn off console logging to control when to display the error
logDisable();
eval
{
$self->check($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $bRequired);
logEnable();
return true;
}
or do
{
# Reset the console logging
logEnable();
# Confess unhandled errors
confess $EVAL_ERROR if (exceptionCode($EVAL_ERROR) != ERROR_BACKUP_MISMATCH);
# Update the DB section if it does not match the current database
$self->dbSectionSet($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $self->dbHistoryIdGet(false)+1);
};
}
}
# Remove backups from FILE_BACKUP_INFO that are no longer in the repository
foreach my $strBackup ($self->keys(INFO_BACKUP_SECTION_BACKUP_CURRENT))
{
my $strManifestFile = "$self->{strBackupClusterPath}/${strBackup}/" . FILE_MANIFEST;
my $strBackupPath = "$self->{strBackupClusterPath}/${strBackup}";
if (!$self->{oStorage}->pathExists($strBackupPath) || !$self->{oStorage}->exists($strManifestFile))
{
&log(WARN, "backup '${strBackup}' missing manifest removed from " . FILE_BACKUP_INFO);
$self->delete($strBackup);
}
}
# ??? Add a section to remove backups that are missing references (unless they are hardlinked?)
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# check
#

52
lib/pgBackRest/Common/Ini.pm
View File

@ -158,38 +158,28 @@ sub loadVersion
my $bCopy = shift;
my $bIgnoreError = shift;
# Make sure the file encryption setting is valid for the repo
if ($self->{oStorage}->encryptionValid($self->{oStorage}->encrypted($self->{strFileName} . ($bCopy ? INI_COPY_EXT : ''),
{bIgnoreMissing => $bIgnoreError})))
# Load main
my $rstrContent = $self->{oStorage}->get(
$self->{oStorage}->openRead($self->{strFileName} . ($bCopy ? INI_COPY_EXT : ''),
{bIgnoreMissing => $bIgnoreError, strCipherPass => $self->{strCipherPass}}));
# If the file exists then attempt to parse it
if (defined($rstrContent))
{
# Load main
my $rstrContent = $self->{oStorage}->get(
$self->{oStorage}->openRead($self->{strFileName} . ($bCopy ? INI_COPY_EXT : ''),
{bIgnoreMissing => $bIgnoreError, strCipherPass => $self->{strCipherPass}}));
my $rhContent = iniParse($$rstrContent, {bIgnoreInvalid => $bIgnoreError});
# If the file exists then attempt to parse it
if (defined($rstrContent))
# If the content is valid then check the header
if (defined($rhContent))
{
my $rhContent = iniParse($$rstrContent, {bIgnoreInvalid => $bIgnoreError});
$self->{oContent} = $rhContent;
# If the content is valid then check the header
if (defined($rhContent))
# If the header is invalid then undef content
if (!$self->headerCheck({bIgnoreInvalid => $bIgnoreError}))
{
$self->{oContent} = $rhContent;
# If the header is invalid then undef content
if (!$self->headerCheck({bIgnoreInvalid => $bIgnoreError}))
{
delete($self->{oContent});
}
delete($self->{oContent});
}
}
}
else
{
confess &log(ERROR, "unable to parse '$self->{strFileName}" . ($bCopy ? INI_COPY_EXT : '') . "'" .
"\nHINT: is or was the repo encrypted?", ERROR_CRYPTO);
}
return defined($self->{oContent});
}
@ -423,10 +413,20 @@ sub save
# Save the file
$self->{oStorage}->put($self->{strFileName}, iniRender($self->{oContent}), {strCipherPass => $self->{strCipherPass}});
$self->{oStorage}->pathSync(dirname($self->{strFileName}));
if ($self->{oStorage}->can('pathSync'))
{
$self->{oStorage}->pathSync(dirname($self->{strFileName}));
}
$self->{oStorage}->put($self->{strFileName} . INI_COPY_EXT, iniRender($self->{oContent}),
{strCipherPass => $self->{strCipherPass}});
$self->{oStorage}->pathSync(dirname($self->{strFileName}));
if ($self->{oStorage}->can('pathSync'))
{
$self->{oStorage}->pathSync(dirname($self->{strFileName}));
}
$self->{bModified} = false;
# Indicate the file now exists

3
lib/pgBackRest/Common/Io/Filter.pm
View File

@ -61,9 +61,6 @@ sub handleWriteSet {shift->{oParent}->handleWriteSet(@_)};
sub name {shift->{oParent}->name()};
sub read {shift->{oParent}->read(@_)};
sub readLine {shift->{oParent}->readLine(@_)};
sub result {shift->{oParent}->result(@_)};
sub resultAll {shift->{oParent}->resultAll()};
sub resultSet {shift->{oParent}->resultSet(@_)};
sub size {shift->{oParent}->size()};
sub timeout {shift->{oParent}->timeout()};
sub write {shift->{oParent}->write(@_)};

32
lib/pgBackRest/Protocol/Storage/Helper.pm
View File

@ -34,38 +34,6 @@ use constant STORAGE_REPO_BACKUP => '<REPO:BA
####################################################################################################################################
my $hStorage;
####################################################################################################################################
# storageDb - get db storage
####################################################################################################################################
sub storageDb
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '::storageDb', \@_,
);
# Create storage if not defined
if (!defined($hStorage->{&STORAGE_DB}))
{
$hStorage->{&STORAGE_DB} = new pgBackRest::Storage::Storage(
STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oStorageDb', value => $hStorage->{&STORAGE_DB}, trace => true},
);
}
push @EXPORT, qw(storageDb);
####################################################################################################################################
# storageRepo - get repository storage
####################################################################################################################################

82
lib/pgBackRest/Storage/Base.pm
View File

@ -8,6 +8,7 @@ use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Digest::SHA qw(sha1_hex);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
@ -27,45 +28,18 @@ use constant STORAGE_S3 => 's3';
use constant STORAGE_POSIX => 'posix';
push @EXPORT, qw(STORAGE_POSIX);
####################################################################################################################################
# Compress constants
####################################################################################################################################
use constant STORAGE_COMPRESS => 'compress';
push @EXPORT, qw(STORAGE_COMPRESS);
use constant STORAGE_DECOMPRESS => 'decompress';
push @EXPORT, qw(STORAGE_DECOMPRESS);
####################################################################################################################################
# Cipher constants
####################################################################################################################################
use constant STORAGE_ENCRYPT => 'encrypt';
push @EXPORT, qw(STORAGE_ENCRYPT);
use constant STORAGE_DECRYPT => 'decrypt';
push @EXPORT, qw(STORAGE_DECRYPT);
use constant CIPHER_MAGIC => 'Salted__';
push @EXPORT, qw(CIPHER_MAGIC);
####################################################################################################################################
# Filter constants
####################################################################################################################################
use constant STORAGE_FILTER_CIPHER_BLOCK => 'pgBackRest::Storage::Filter::CipherBlock';
push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);
use constant STORAGE_FILTER_GZ => 'pgBackRest::Storage::Filter::Gz';
push @EXPORT, qw(STORAGE_FILTER_GZ);
####################################################################################################################################
# Capability constants
####################################################################################################################################
# Can the size in the storage be different than what was written? For example, a ZFS filesystem could be doing compression of a
# backup where compression was not enabled. This affects how repo-size is calculated. If the file system only stores what was
# written or won't report differently then we can save some time by just setting repo-size to size.
use constant STORAGE_CAPABILITY_SIZE_DIFF => 'size-diff';
push @EXPORT, qw(STORAGE_CAPABILITY_SIZE_DIFF);
# Does the storage support symlinks and hardlinks?
use constant STORAGE_CAPABILITY_LINK => 'link';
push @EXPORT, qw(STORAGE_CAPABILITY_LINK);
use constant STORAGE_CAPABILITY_PATH_SYNC => 'path-sync';
push @EXPORT, qw(STORAGE_CAPABILITY_PATH_SYNC);
####################################################################################################################################
# new
@ -236,6 +210,58 @@ sub get
);
}
####################################################################################################################################
# Calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from openRead/openWrite
# must be passed instead of file names.
####################################################################################################################################
sub hashSize
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->hashSize', \@_,
{name => 'xFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Set operation variables
my $strHash;
my $lSize;
# Is this an IO object or a file expression?
my $rtContent = $self->get($xFileExp, {bIgnoreMissing => $bIgnoreMissing});
if (defined($rtContent))
{
if (defined($$rtContent))
{
$strHash = sha1_hex($$rtContent);
$lSize = length($$rtContent);
}
else
{
$strHash = sha1_hex('');
$lSize = 0;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strHash', value => $strHash},
{name => 'lSize', value => $lSize}
);
}
####################################################################################################################################
# pathAbsolute - generate an absolute path from an absolute base path and a relative path
####################################################################################################################################

532
lib/pgBackRest/Storage/Storage.pm
View File

@ -92,14 +92,11 @@ sub exists
{name => 'strFileExp'},
);
# Check exists
my $bExists = $self->{oStorageC}->exists($strFileExp);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $bExists ? true : false}
{name => 'bExists', value => defined($self->info($strFileExp, {bIgnoreMissing => true}))}
);
}
@ -146,58 +143,6 @@ sub get
);
}
####################################################################################################################################
# Calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from openRead/openWrite
# must be passed instead of file names.
####################################################################################################################################
sub hashSize
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->hashSize', \@_,
{name => 'xFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Set operation variables
my $strHash;
my $lSize;
# Is this an IO object or a file expression?
my $rtContent = $self->get($xFileExp, {bIgnoreMissing => $bIgnoreMissing});
if (defined($rtContent))
{
if (defined($$rtContent))
{
$strHash = sha1_hex($$rtContent);
$lSize = length($$rtContent);
}
else
{
$strHash = sha1_hex('');
$lSize = 0;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strHash', value => $strHash},
{name => 'lSize', value => $lSize}
);
}
####################################################################################################################################
# Get information for path/file
####################################################################################################################################
@ -235,112 +180,6 @@ sub info
);
}
####################################################################################################################################
# linkCreate - create a link
####################################################################################################################################
sub linkCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourcePathFileExp,
$strDestinationLinkExp,
$bHard,
$bRelative,
$bPathCreate,
$bIgnoreExists,
) =
logDebugParam
(
__PACKAGE__ . '->linkCreate', \@_,
{name => 'strSourcePathFileExp'},
{name => 'strDestinationLinkExp'},
{name => 'bHard', optional=> true, default => false},
{name => 'bRelative', optional=> true, default => false},
{name => 'bPathCreate', optional=> true, default => true},
{name => 'bIgnoreExists', optional => true, default => false},
);
# Get source and destination paths
my $strSourcePathFile = $self->pathGet($strSourcePathFileExp);
my $strDestinationLink = $self->pathGet($strDestinationLinkExp);
# Generate relative path if requested
if ($bRelative)
{
# Determine how much of the paths are common
my @strySource = split('/', $strSourcePathFile);
my @stryDestination = split('/', $strDestinationLink);
while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0])
{
shift(@strySource);
shift(@stryDestination);
}
# Add relative path sections
$strSourcePathFile = '';
for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++)
{
$strSourcePathFile .= '../';
}
# Add path to source
$strSourcePathFile .= join('/', @strySource);
logDebugMisc
(
$strOperation, 'apply relative path',
{name => 'strSourcePathFile', value => $strSourcePathFile, trace => true}
);
}
if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink)))
{
my $strMessage = "unable to create link '${strDestinationLink}'";
# If parent path or source is missing
if ($OS_ERROR{ENOENT})
{
# Check if source is missing
if (!$self->exists($strSourcePathFile))
{
confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING);
}
if (!$bPathCreate)
{
confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING);
}
# Create parent path
$self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true});
# Create link
$self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard});
}
# Else if link already exists
elsif ($OS_ERROR{EEXIST})
{
if (!$bIgnoreExists)
{
confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS);
}
}
else
{
logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);
}
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# List all files/paths in path
####################################################################################################################################
@ -442,42 +281,6 @@ sub manifestJson
);
}
####################################################################################################################################
# move - move path/file
####################################################################################################################################
sub move
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourceFileExp,
$strDestinationFileExp,
$bPathCreate,
) =
logDebugParam
(
__PACKAGE__ . '->move', \@_,
{name => 'strSourceFileExp'},
{name => 'strDestinationFileExp'},
);
# Get source and destination paths
my $strSourceFile = $self->pathGet($strSourceFileExp);
my $strDestinationFile = $self->pathGet($strDestinationFileExp);
# Move the file
if (!rename($strSourceFile, $strDestinationFile))
{
logErrorResult(ERROR_FILE_MOVE, "unable to move '${strSourceFile}' to '${strDestinationFile}'", $OS_ERROR);
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Open file for reading
####################################################################################################################################
@ -560,7 +363,7 @@ sub openWrite
{name => 'strGroup', optional => true},
{name => 'lTimestamp', optional => true, default => '0'},
{name => 'bAtomic', optional => true, default => false},
{name => 'bPathCreate', optional => true, default => false},
{name => 'bPathCreate', optional => true, default => true},
{name => 'rhyFilter', optional => true},
{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},
);
@ -593,226 +396,6 @@ sub openWrite
);
}
####################################################################################################################################
# Change ownership of path/file
####################################################################################################################################
sub owner
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathFileExp,
$strUser,
$strGroup
) =
logDebugParam
(
__PACKAGE__ . '->owner', \@_,
{name => 'strPathFileExp'},
{name => 'strUser', required => false},
{name => 'strGroup', required => false}
);
# Only proceed if user or group was specified
if (defined($strUser) || defined($strGroup))
{
my $strPathFile = $self->pathGet($strPathFileExp);
my $strMessage = "unable to set ownership for '${strPathFile}'";
my $iUserId;
my $iGroupId;
# If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that
# both user and group be set.
my $oStat = lstat($strPathFile);
if (!defined($oStat))
{
confess &log(ERROR, "unable to stat '${strPathFile}': No such file or directory", ERROR_FILE_MISSING);
}
if (!defined($strUser))
{
$iUserId = $oStat->uid;
}
if (!defined($strGroup))
{
$iGroupId = $oStat->gid;
}
# Lookup user if specified
if (defined($strUser))
{
$iUserId = getpwnam($strUser);
if (!defined($iUserId))
{
logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist");
}
}
# Lookup group if specified
if (defined($strGroup))
{
$iGroupId = getgrnam($strGroup);
if (!defined($iGroupId))
{
logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist");
}
}
# Set ownership on the file if the user or group would be changed
if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid)
{
if (!chown($iUserId, $iGroupId, $strPathFile))
{
logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR);
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# Generate an absolute path from an absolute base path and a relative path
####################################################################################################################################
sub pathAbsolute
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strBasePath,
$strPath
) =
logDebugParam
(
__PACKAGE__ . '->pathAbsolute', \@_,
{name => 'strBasePath', trace => true},
{name => 'strPath', trace => true}
);
# Working variables
my $strAbsolutePath;
# If the path is already absolute
if (index($strPath, '/') == 0)
{
$strAbsolutePath = $strPath;
}
# Else make it absolute using the base path
else
{
# Make sure the absolute path is really absolute
if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)
{
confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE);
}
while (index($strPath, '..') == 0)
{
$strBasePath = dirname($strBasePath);
$strPath = substr($strPath, 2);
if (index($strPath, '/') == 0)
{
$strPath = substr($strPath, 1);
}
}
$strAbsolutePath = "${strBasePath}/${strPath}";
}
# Make sure the result is really an absolute path
if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)
{
confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}
);
}
####################################################################################################################################
# Create a path
####################################################################################################################################
sub pathCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$strMode,
$bIgnoreExists,
$bCreateParent,
) =
logDebugParam
(
__PACKAGE__ . '->pathCreate', \@_,
{name => 'strPathExp'},
{name => 'strMode', optional => true},
{name => 'bIgnoreExists', optional => true, default => false},
{name => 'bCreateParent', optional => true, default => false},
);
# Create path
$self->{oStorageC}->pathCreate($strPathExp, $strMode, $bIgnoreExists, $bCreateParent);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# Check if path exists
####################################################################################################################################
sub pathExists
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->pathExists', \@_,
{name => 'strPathExp'},
);
# Check exists
my $bExists = $self->{oStorageC}->pathExists($strPathExp);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $bExists ? true : false}
);
}
####################################################################################################################################
# Resolve a path expression into an absolute path
####################################################################################################################################
@ -872,31 +455,6 @@ sub pathRemove
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Sync path so newly added file entries are not lost
####################################################################################################################################
sub pathSync
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->pathSync', \@_,
{name => 'strPathExp'},
);
$self->{oStorageC}->pathSync($strPathExp);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# put - writes a buffer out to storage all at once
####################################################################################################################################
@ -965,92 +523,6 @@ sub remove
return logDebugReturn($strOperation);
}
####################################################################################################################################
# encrypted - determine if the file is encrypted or not
####################################################################################################################################
sub encrypted
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileExp,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->encrypted', \@_,
{name => 'strFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
my $bEncrypted = false;
# Open the file via the driver
my $oFileIo = new pgBackRest::Storage::StorageRead(
$self, pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $strFileExp, $bIgnoreMissing));
# If the file does not exist because we're ignoring missing (else it would error before this is executed) then determine if it
# should be encrypted based on the repo
if (!$oFileIo->open())
{
if (defined($self->cipherType()))
{
$bEncrypted = true;
}
}
else
{
# If the file does exist, then read the magic signature
my $tMagicSignature = '';
my $lSizeRead = $oFileIo->read(\$tMagicSignature, length(CIPHER_MAGIC));
$oFileIo->close();
if (substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC)
{
$bEncrypted = true;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bEncrypted', value => $bEncrypted}
);
}
####################################################################################################################################
# encryptionValid - determine if encryption set properly based on the value passed
####################################################################################################################################
sub encryptionValid
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bEncrypted,
) =
logDebugParam
(
__PACKAGE__ . '->encryptionValid', \@_,
{name => 'bEncrypted'},
);
my $bValid = ($bEncrypted && defined($self->cipherType())) || (!$bEncrypted && !defined($self->cipherType()));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bValid', value => $bValid ? true : false}
);
}
####################################################################################################################################
# Getters
####################################################################################################################################

47
lib/pgBackRest/Storage/StorageRead.pm
View File

@ -140,51 +140,4 @@ sub close
);
}
####################################################################################################################################
# Get a filter result
####################################################################################################################################
sub result
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strClass,
) =
logDebugParam
(
__PACKAGE__ . '->result', \@_,
{name => 'strClass'},
);
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->result($strClass));
return logDebugReturn
(
$strOperation,
{name => 'xResult', value => $xResult, trace => true},
);
}
####################################################################################################################################
# Get all filter results
####################################################################################################################################
sub resultAll
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->resultAll());
return logDebugReturn
(
$strOperation,
{name => 'xResultAll', value => $xResult, trace => true},
);
}
1;

47
lib/pgBackRest/Storage/StorageWrite.pm
View File

@ -113,51 +113,4 @@ sub close
);
}
####################################################################################################################################
# Get a filter result
####################################################################################################################################
sub result
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strClass,
) =
logDebugParam
(
__PACKAGE__ . '->result', \@_,
{name => 'strClass'},
);
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->result($strClass));
return logDebugReturn
(
$strOperation,
{name => 'xResult', value => $xResult, trace => true},
);
}
####################################################################################################################################
# Get all filter results
####################################################################################################################################
sub resultAll
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');
my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->resultAll());
return logDebugReturn
(
$strOperation,
{name => 'xResultAll', value => $xResult, trace => true},
);
}
1;

73
libc/xs/storage/storage.xs
View File

@ -83,23 +83,6 @@ CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
exists(self, fileExp)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *fileExp = STR_NEW_SV($arg);
CODE:
RETVAL = storageExistsP(self, fileExp);
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
get(self, read)
@ -215,47 +198,6 @@ CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
pathCreate(self, pathExp, mode, ignoreExists, createParent)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
const String *mode = STR_NEW_SV($arg);
bool ignoreExists
bool createParent
CODE:
if (storageFeature(self, storageFeaturePath))
storagePathCreateP(
self, pathExp, .mode = mode ? cvtZToMode(strPtr(mode)) : 0, .errorOnExists = !ignoreExists,
.noParentCreate = !createParent);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
pathExists(self, pathExp)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
CODE:
RETVAL = true;
if (storageFeature(self, storageFeaturePath))
RETVAL = storagePathExistsP(self, pathExp);
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
pathGet(self, pathExp)
@ -292,21 +234,6 @@ CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
pathSync(self, pathExp)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
CODE:
storagePathSyncP(self, pathExp);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
UV
put(self, write, buffer)

66
libc/xs/storage/storage.xsh
View File

@ -124,72 +124,6 @@ storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const Strin
varUInt64Force(varLstGet(paramList, 0)) ? cipherModeEncrypt : cipherModeDecrypt,
cipherType(varStr(varLstGet(paramList, 1))), BUFSTR(varStr(varLstGet(paramList, 2))), NULL));
}
else if (strEqZ(filter, "pgBackRest::Common::Io::Handle"))
{
ioFilterGroupAdd(filterGroup, ioSizeNew());
}
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gz"))
{
if (strEqZ(varStr(varLstGet(paramList, 0)), "compress"))
ioFilterGroupAdd(filterGroup, gzCompressNew(varUIntForce(varLstGet(paramList, 1))));
else
ioFilterGroupAdd(filterGroup, gzDecompressNew());
}
else
THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter));
}
/***********************************************************************************************************************************
Get result from IO filter
***********************************************************************************************************************************/
String *
storageFilterXsResult(const IoFilterGroup *filterGroup, const String *filter)
{
const Variant *result;
if (strEqZ(filter, "pgBackRest::Common::Io::Handle"))
{
result = ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR);
}
else
THROW_FMT(AssertError, "unable to get result for invalid filter '%s'", strPtr(filter));
if (result == NULL)
THROW_FMT(AssertError, "unable to find result for filter '%s'", strPtr(filter));
return jsonFromVar(result);
}
/***********************************************************************************************************************************
Get results from all IO filters
***********************************************************************************************************************************/
String *
storageFilterXsResultAll(const IoFilterGroup *filterGroup)
{
const VariantList *filterList = kvKeyList(varKv(ioFilterGroupResultAll(filterGroup)));
String *result = strNew("{");
for (unsigned int filterIdx = 0; filterIdx < varLstSize(filterList); filterIdx++)
{
const String *filter = varStr(varLstGet(filterList, filterIdx));
const String *filterPerl = NULL;
if (strEq(filter, SIZE_FILTER_TYPE_STR))
{
filterPerl = strNew("pgBackRest::Common::Io::Handle");
}
if (filterPerl != NULL)
{
if (strSize(result) > 1)
strCat(result, ",");
strCatFmt(
result, "%s:%s", strPtr(jsonFromStr(filterPerl)), strPtr(storageFilterXsResult(filterGroup, filterPerl)));
}
}
strCat(result, "}");
return result;
}

33
libc/xs/storage/storageRead.xs
View File

@ -111,39 +111,6 @@ CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
result(self, filter)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
const String *filter = STR_NEW_SV($arg);
CODE:
RETVAL = strPtr(storageFilterXsResult(ioReadFilterGroup(storageReadIo(self)), filter));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
resultAll(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
CODE:
RETVAL = strPtr(storageFilterXsResultAll(ioReadFilterGroup(storageReadIo(self))));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
DESTROY(self)

33
libc/xs/storage/storageWrite.xs
View File

@ -98,39 +98,6 @@ CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
result(self, filter)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
const String *filter = STR_NEW_SV($arg);
CODE:
RETVAL = strPtr(storageFilterXsResult(ioWriteFilterGroup(storageWriteIo(self)), filter));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
resultAll(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
CODE:
RETVAL = strPtr(storageFilterXsResultAll(ioWriteFilterGroup(storageWriteIo(self))));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
DESTROY(self)

2
test/lib/pgBackRestTest/Common/FileTest.pm
View File

@ -168,7 +168,7 @@ sub forceStorageMode
);
# Mode commands are ignored on S3
if ($oStorage->type() ne STORAGE_S3)
if (!$oStorage->can('type') || $oStorage->type() ne STORAGE_S3)
{
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
}

7
test/lib/pgBackRestTest/Common/RunTest.pm
View File

@ -19,14 +19,14 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Storage;
use pgBackRest::Version;
use pgBackRestTest::Common::BuildTest;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::LogTest;
use pgBackRestTest::Common::Storage;
use pgBackRestTest::Common::StoragePosix;
use pgBackRestTest::Common::VmTest;
####################################################################################################################################
@ -156,7 +156,8 @@ sub process
$self->{bFirstTest} = true;
# Initialize test storage
$oStorage = new pgBackRest::Storage::Storage(STORAGE_LOCAL, {strPath => $self->testPath()});
$oStorage = new pgBackRestTest::Common::Storage(
$self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Generate backrest exe
$self->{strBackRestExe} = defined($self->{strBackRestExeC}) ? $self->{strBackRestExeC} : $self->{strBackRestExeHelper};

15
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
View File

@ -221,7 +221,7 @@ sub stanzaSet
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false, false,
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Else get the info data from disk
@ -260,7 +260,6 @@ sub stanzaSet
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
storageRepo()->pathCreate($$oStanza{strArchiveClusterPath}, {bCreateParent => true});
$self->{oStanzaHash}{$strStanza} = $oStanza;
@ -301,15 +300,6 @@ sub stanzaCreate
$self->controlGenerate($strDbPath, $strDbVersion);
executeTest('chmod 600 ' . $strDbPath . '/' . DB_FILE_PGCONTROL);
# Create the stanza repo paths if they don't exist
if (!cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3))
{
storageTest()->pathCreate(
cfgOption(CFGOPT_REPO_PATH) . "/archive/$strStanza", {bIgnoreExists => true, bCreateParent => true});
storageTest()->pathCreate(
cfgOption(CFGOPT_REPO_PATH) . "/backup/$strStanza", {bIgnoreExists => true, bCreateParent => true});
}
# Create the stanza and set the local stanza object
$self->stanzaSet($strStanza, $strDbVersion, false);
@ -398,7 +388,6 @@ sub backupCreate
$lTimestamp);
my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}";
storageRepo()->pathCreate($strBackupClusterSetPath);
&log(INFO, "create backup ${strBackupLabel}");
@ -558,8 +547,6 @@ sub archiveCreate
do
{
my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16);
storageRepo()->pathCreate($strPath, {bIgnoreExists => true});
my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : '');
storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass});

13
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@ -102,12 +102,6 @@ sub new
$self->{strRepoPath} = '/';
}
# Create the repo-path if on a local filesystem
if ($$oParam{strBackupDestination} eq $self->nameGet() && $oParam->{bRepoLocal})
{
storageTest()->pathCreate($self->repoPath(), {strMode => '0770'});
}
# Set log/lock paths
$self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG;
storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'});
@ -309,8 +303,11 @@ sub backupEnd
}
}
}
# Else there should not be a tablespace directory at all
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
# Else there should not be a tablespace directory at all. This is only valid for storage that supports links.
elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) &&
storageTest()->pathExists(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}

6
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -142,7 +142,7 @@ sub run
'184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true);
# Load sample page
my $tBasePage = ${storageLocal()->get($self->dataPath() . '/page.bin')};
my $tBasePage = ${storageTest()->get($self->dataPath() . '/page.bin')};
my $iBasePageChecksum = 0x1B99;
# Create base path
@ -477,7 +477,7 @@ sub run
strOptionalParam => '--force --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . ($bDeltaBackup ? ' --delta' : '')});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta)
storageDb->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
storageTest->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
# Restore - tests various mode, extra files/paths, missing files/paths
#---------------------------------------------------------------------------------------------------------------------------
@ -680,7 +680,7 @@ sub run
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changetime.txt'}{&MANIFEST_SUBKEY_TIMESTAMP} = $lTime - 100;
# Change the content of the changecontent file to be the same size but leave the timestamp the same on the file
storageDb()->put($oHostDbMaster->dbBasePath() . '/changecontent.txt', 'CHGCONT');
storageTest()->put($oHostDbMaster->dbBasePath() . '/changecontent.txt', 'CHGCONT');
utime($lTime, $lTime, $oHostDbMaster->dbBasePath() . '/changecontent.txt')
or confess &log(ERROR, "unable to set time for file ".$oHostDbMaster->dbBasePath() . '/changecontent.txt');

10
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@ -180,9 +180,9 @@ sub run
$strCommandGet . " ${strSourceFile} ${strWalPath}/RECOVERYXLOG", {oLogTest => $self->expect()});
# Check that the destination file exists
if (storageDb()->exists("${strWalPath}/RECOVERYXLOG"))
if (storageTest()->exists("${strWalPath}/RECOVERYXLOG"))
{
my ($strActualChecksum) = storageDb()->hashSize("${strWalPath}/RECOVERYXLOG");
my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG");
if ($strActualChecksum ne $strArchiveChecksum)
{
@ -344,9 +344,9 @@ sub run
{oLogTest => $self->expect()});
# Check that the destination file exists
if (storageDb()->exists("${strWalPath}/RECOVERYXLOG"))
if (storageTest()->exists("${strWalPath}/RECOVERYXLOG"))
{
my ($strActualChecksum) = storageDb()->hashSize("${strWalPath}/RECOVERYXLOG");
my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG");
if ($strActualChecksum ne $strArchiveChecksum)
{
@ -369,7 +369,7 @@ sub run
$strCommandGet . " --archive-async 00000002.history ${strWalPath}/00000002.history",
{oLogTest => $self->expect()});
if (${storageDb()->get("${strWalPath}/00000002.history")} ne 'HISTORYDATA')
if (${storageTest()->get("${strWalPath}/00000002.history")} ne 'HISTORYDATA')
{
confess 'history contents do not match original';
}

10
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
View File

@ -83,7 +83,7 @@ sub run
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'});
# Generate pg_control for stanza-create
storageDb()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
storageTest()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93);
# Fail stanza upgrade before stanza-create has been performed
@ -119,7 +119,7 @@ sub run
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
storageDb()->pathCreate("${strWalPath}/archive_status", {bCreateParent => true});
storageTest()->pathCreate("${strWalPath}/archive_status", {bCreateParent => true});
# Stanza Create fails - missing archive.info from non-empty archive dir
#--------------------------------------------------------------------------------------------------------------------------
@ -158,7 +158,7 @@ sub run
# Upgrade the DB by copying new pg_control
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# Fail on attempt to push an archive
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1, ERROR_ARCHIVE_MISMATCH);
@ -184,7 +184,7 @@ sub run
# Copy the new pg_control back so the tests can continue with the upgraded stanza
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# After stanza upgrade, make sure archives are pushed to the new db verion-id directory (9.4-2)
#--------------------------------------------------------------------------------------------------------------------------
@ -206,7 +206,7 @@ sub run
#--------------------------------------------------------------------------------------------------------------------------
# Copy pg_control for 9.5
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});

14
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -324,7 +324,7 @@ sub run
'fail on backup info file missing from non-empty dir', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
# Change the database version by copying a new pg_control file to a new pg-path to use for db mismatch test
storageLocal()->pathCreate(
storageTest()->pathCreate(
$oHostDbMaster->dbPath() . '/testbase/' . DB_PATH_GLOBAL,
{strMode => '0700', bIgnoreExists => true, bCreateParent => true});
$self->controlGenerate(
@ -774,7 +774,7 @@ sub run
# Version <= 8.4 always places a PG_VERSION file in the tablespace
if ($oHostDbMaster->pgVersion() <= PG_VERSION_84)
{
if (!storageLocal()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
if (!storageTest()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
{
confess &log(ASSERT, "unable to find '" . DB_FILE_PGVERSION . "' in tablespace path '${strTablespacePath}'");
}
@ -792,14 +792,14 @@ sub run
'/PG_' . $oHostDbMaster->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG);
# Check that path exists
if (!storageLocal()->pathExists($strTablespacePath))
if (!storageTest()->pathExists($strTablespacePath))
{
confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'");
}
}
# Make sure there are some files in the tablespace path (exclude PG_VERSION if <= 8.4 since that was tested above)
if (grep(!/^PG\_VERSION$/i, storageLocal()->list($strTablespacePath)) == 0)
if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0)
{
confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'");
}
@ -869,7 +869,7 @@ sub run
# Save recovery file to test so we can use it in the next test
$strRecoveryFile = $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF;
storageLocal()->copy(
storageTest()->copy(
$oHostDbMaster->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile);
$oHostDbMaster->clusterStart();
@ -891,12 +891,12 @@ sub run
executeTest('rm -rf ' . $oHostDbMaster->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
storageLocal()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
# Also touch recovery.signal when required
if ($oHostDbMaster->pgVersion() >= PG_VERSION_12)
{
storageDb()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
storageTest()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
}
$oHostDbMaster->restore(