From 0a96a2895d7a1e4c5eec402d3742f1d1e25cc126 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 17 Jun 2019 09:16:44 -0400 Subject: [PATCH] Add storage layer for tests and documentation. The tests and documentation have been using the core storage layer but soon that will depend entirely on the C library, creating a bootstrap problem (i.e. the storage layer will be needed to build the C library). Create a simplified Posix storage layer to be used by documentation and the parts of the test code that build and execute the actual tests. The actual tests will still use the core storage driver so they can interact with any type of storage. --- build/lib/pgBackRestBuild/Build.pm | 9 +- .../lib/pgBackRestBuild/Config/BuildDefine.pm | 4 +- doc/doc.pl | 8 +- .../BackRestDoc/Custom/DocCustomRelease.pm | 4 +- doc/release.pl | 8 +- test/lib/pgBackRestTest/Common/Storage.pm | 653 ++++++++++++ .../lib/pgBackRestTest/Common/StoragePosix.pm | 982 ++++++++++++++++++ .../pgBackRestTest/Common/StoragePosixRead.pm | 105 ++ .../Common/StoragePosixWrite.pm | 209 ++++ test/test.pl | 12 +- 10 files changed, 1972 insertions(+), 22 deletions(-) create mode 100644 test/lib/pgBackRestTest/Common/Storage.pm create mode 100644 test/lib/pgBackRestTest/Common/StoragePosix.pm create mode 100644 test/lib/pgBackRestTest/Common/StoragePosixRead.pm create mode 100644 test/lib/pgBackRestTest/Common/StoragePosixWrite.pm diff --git a/build/lib/pgBackRestBuild/Build.pm b/build/lib/pgBackRestBuild/Build.pm index 9aed688d2..dd6a54232 100644 --- a/build/lib/pgBackRestBuild/Build.pm +++ b/build/lib/pgBackRestBuild/Build.pm @@ -15,11 +15,12 @@ use Storable qw(dclone); use pgBackRest::Common::Log; use pgBackRest::Common::String; -use pgBackRest::Storage::Local; -use pgBackRest::Storage::Posix::Driver; use pgBackRestBuild::Build::Common; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; + #################################################################################################################################### # Define generator used for auto generated warning messages #################################################################################################################################### @@ -35,8 +36,8 @@ sub buildAll my $strFileExt = shift; # Storage object - my $oStorage = new pgBackRest::Storage::Local( - $strBuildPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorage = new pgBackRestTest::Common::Storage( + $strBuildPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); # List of files actually built my @stryBuilt; diff --git a/build/lib/pgBackRestBuild/Config/BuildDefine.pm b/build/lib/pgBackRestBuild/Config/BuildDefine.pm index f3715c94c..c44f68b45 100644 --- a/build/lib/pgBackRestBuild/Config/BuildDefine.pm +++ b/build/lib/pgBackRestBuild/Config/BuildDefine.pm @@ -350,8 +350,8 @@ sub buildConfigDefine my $strDocPath = abs_path(dirname($0) . '/../doc'); - my $oStorageDoc = new pgBackRest::Storage::Local( - $strDocPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageDoc = new pgBackRestTest::Common::Storage( + $strDocPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); my @stryEmpty = []; my $oManifest = new BackRestDoc::Common::DocManifest( diff --git a/doc/doc.pl b/doc/doc.pl index da6b891de..e53143332 100755 --- a/doc/doc.pl +++ b/doc/doc.pl @@ -35,11 +35,11 @@ use BackRestDoc::Markdown::DocMarkdown; use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::String; -use pgBackRest::Storage::Local; -use pgBackRest::Storage::Posix::Driver; use pgBackRest::Version; use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; #################################################################################################################################### # Usage @@ -193,8 +193,8 @@ eval # Get the base path my $strBasePath = abs_path(dirname($0)); - my $oStorageDoc = new pgBackRest::Storage::Local( - $strBasePath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageDoc = new pgBackRestTest::Common::Storage( + $strBasePath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); if (!defined($strDocPath)) { diff --git a/doc/lib/BackRestDoc/Custom/DocCustomRelease.pm b/doc/lib/BackRestDoc/Custom/DocCustomRelease.pm index 35c26b883..75e4f8f4f 100644 --- a/doc/lib/BackRestDoc/Custom/DocCustomRelease.pm +++ b/doc/lib/BackRestDoc/Custom/DocCustomRelease.pm @@ -316,8 +316,8 @@ sub docGet my $strOperation = logDebugParam(__PACKAGE__ . '->docGet'); # Load the git history - my $oStorageDoc = new pgBackRest::Storage::Local( - dirname(abs_path($0)), new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageDoc = new pgBackRestTest::Common::Storage( + dirname(abs_path($0)), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); my @hyGitLog = @{(JSON::PP->new()->allow_nonref())->decode(${$oStorageDoc->get("resource/git-history.cache")})}; # Get renderer diff --git a/doc/release.pl b/doc/release.pl index bd3d122bb..6954e5dc5 100755 --- a/doc/release.pl +++ b/doc/release.pl @@ -35,11 +35,11 @@ use BackRestDoc::Markdown::DocMarkdown; use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::String; -use pgBackRest::Storage::Local; -use pgBackRest::Storage::Posix::Driver; use pgBackRest::Version; use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; #################################################################################################################################### # Usage @@ -124,8 +124,8 @@ eval my $strDocExe = "${strDocPath}/doc.pl"; my $strTestExe = dirname($strDocPath) . "/test/test.pl"; - my $oStorageDoc = new pgBackRest::Storage::Local( - $strDocPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageDoc = new pgBackRestTest::Common::Storage( + $strDocPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); # Determine if this is a dev release my $bDev = PROJECT_VERSION =~ /dev$/; diff --git a/test/lib/pgBackRestTest/Common/Storage.pm b/test/lib/pgBackRestTest/Common/Storage.pm new file mode 100644 index 000000000..a15b1d0a3 --- /dev/null +++ b/test/lib/pgBackRestTest/Common/Storage.pm @@ -0,0 +1,653 @@ +#################################################################################################################################### +# Implements storage functionality using drivers. +#################################################################################################################################### +package pgBackRestTest::Common::Storage; +use parent 'pgBackRest::Storage::Base'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use File::Basename qw(dirname); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Common::String; +use pgBackRest::Storage::Base; + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathBase, + $oDriver, + $bAllowTemp, + $strTempExtension, + $strDefaultPathMode, + $strDefaultFileMode, + $lBufferMax, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'strPathBase'}, + {name => 'oDriver'}, + {name => 'bAllowTemp', optional => true, default => true}, + {name => 'strTempExtension', optional => true, default => 'tmp'}, + {name => 'strDefaultPathMode', optional => true, default => '0750'}, + {name => 'strDefaultFileMode', optional => true, default => '0640'}, + {name => 'lBufferMax', optional => true}, + ); + + # Create class + my $self = $class->SUPER::new({lBufferMax => $lBufferMax}); + bless $self, $class; + + $self->{strPathBase} = $strPathBase; + $self->{oDriver} = $oDriver; + $self->{bAllowTemp} = $bAllowTemp; + $self->{strTempExtension} = $strTempExtension; + $self->{strDefaultPathMode} = $strDefaultPathMode; + $self->{strDefaultFileMode} = $strDefaultFileMode; + + # Set temp extension in driver + $self->driver()->tempExtensionSet($self->{strTempExtension}) if $self->driver()->can('tempExtensionSet'); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# exists - check if file exists +#################################################################################################################################### +sub exists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFileExp, + ) = + logDebugParam + ( + __PACKAGE__ . '->exists', \@_, + {name => 'strFileExp'}, + ); + + # Check exists + my $bExists = $self->driver()->exists($self->pathGet($strFileExp)); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists} + ); +} + +#################################################################################################################################### +# info - get information for path/file +#################################################################################################################################### +sub info +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathFileExp, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '::fileStat', \@_, + {name => 'strPathFileExp'}, + {name => 'bIgnoreMissing', optional => true, default => false}, + ); + + # Stat the path/file + my $oInfo = $self->driver()->info($self->pathGet($strPathFileExp), {bIgnoreMissing => $bIgnoreMissing}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oInfo', value => $oInfo, trace => true} + ); +} + +#################################################################################################################################### +# linkCreate - create a link +#################################################################################################################################### +sub linkCreate +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strSourcePathFileExp, + $strDestinationLinkExp, + $bHard, + $bRelative, + $bPathCreate, + $bIgnoreExists, + ) = + logDebugParam + ( + __PACKAGE__ . '->linkCreate', \@_, + {name => 'strSourcePathFileExp'}, + {name => 'strDestinationLinkExp'}, + {name => 'bHard', optional=> true, default => false}, + {name => 'bRelative', optional=> true, default => false}, + {name => 'bPathCreate', optional=> true, default => true}, + {name => 'bIgnoreExists', optional => true, default => false}, + ); + + # Get source and destination paths + my $strSourcePathFile = $self->pathGet($strSourcePathFileExp); + my $strDestinationLink = $self->pathGet($strDestinationLinkExp); + + # Generate relative path if requested + if ($bRelative) + { + # Determine how much of the paths are common + my @strySource = split('/', $strSourcePathFile); + my @stryDestination = split('/', $strDestinationLink); + + while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0]) + { + shift(@strySource); + shift(@stryDestination); + } + + # Add relative path sections + $strSourcePathFile = ''; + + for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++) + { + $strSourcePathFile .= '../'; + } + + # Add path to source + $strSourcePathFile .= join('/', @strySource); + + logDebugMisc + ( + $strOperation, 'apply relative path', + {name => 'strSourcePathFile', value => $strSourcePathFile, trace => true} + ); + } + + # Create the link + $self->driver()->linkCreate( + $strSourcePathFile, $strDestinationLink, {bHard => $bHard, bPathCreate => $bPathCreate, bIgnoreExists => $bIgnoreExists}); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# list - list all files/paths in path +#################################################################################################################################### +sub list +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + $strExpression, + $strSortOrder, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->list', \@_, + {name => 'strPathExp', required => false}, + {name => 'strExpression', optional => true}, + {name => 'strSortOrder', optional => true, default => 'forward'}, + {name => 'bIgnoreMissing', optional => true, default => false}, + ); + + # Get file list + my $rstryFileList = $self->driver()->list( + $self->pathGet($strPathExp), {strExpression => $strExpression, bIgnoreMissing => $bIgnoreMissing}); + + # Apply expression if defined + if (defined($strExpression)) + { + @{$rstryFileList} = grep(/$strExpression/i, @{$rstryFileList}); + } + + # Reverse sort + if ($strSortOrder eq 'reverse') + { + @{$rstryFileList} = sort {$b cmp $a} @{$rstryFileList}; + } + # Normal sort + else + { + @{$rstryFileList} = sort @{$rstryFileList}; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'stryFileList', value => $rstryFileList} + ); +} + +#################################################################################################################################### +# manifest - build path/file/link manifest starting with base path and including all subpaths +#################################################################################################################################### +sub manifest +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + $strFilter, + ) = + logDebugParam + ( + __PACKAGE__ . '->manifest', \@_, + {name => 'strPathExp'}, + {name => 'strFilter', optional => true, trace => true}, + ); + + my $hManifest = $self->driver()->manifest($self->pathGet($strPathExp), {strFilter => $strFilter}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hManifest', value => $hManifest, trace => true} + ); +} + +#################################################################################################################################### +# move - move path/file +#################################################################################################################################### +sub move +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strSourcePathFileExp, + $strDestinationPathFileExp, + $bPathCreate, + ) = + logDebugParam + ( + __PACKAGE__ . '->move', \@_, + {name => 'strSourcePathExp'}, + {name => 'strDestinationPathExp'}, + {name => 'bPathCreate', optional => true, default => false, trace => true}, + ); + + # Set operation variables + $self->driver()->move( + $self->pathGet($strSourcePathFileExp), $self->pathGet($strDestinationPathFileExp), {bPathCreate => $bPathCreate}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation + ); +} + +#################################################################################################################################### +# openRead - open file for reading +#################################################################################################################################### +sub openRead +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFileExp, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->openRead', \@_, + {name => 'xFileExp'}, + {name => 'bIgnoreMissing', optional => true, default => false}, + ); + + # Open the file + my $oFileIo = $self->driver()->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIo', value => $oFileIo, trace => true}, + ); +} + +#################################################################################################################################### +# openWrite - open file for writing +#################################################################################################################################### +sub openWrite +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFileExp, + $strMode, + $strUser, + $strGroup, + $lTimestamp, + $bAtomic, + $bPathCreate, + ) = + logDebugParam + ( + __PACKAGE__ . '->openWrite', \@_, + {name => 'xFileExp'}, + {name => 'strMode', optional => true, default => $self->{strDefaultFileMode}}, + {name => 'strUser', optional => true}, + {name => 'strGroup', optional => true}, + {name => 'lTimestamp', optional => true}, + {name => 'bAtomic', optional => true, default => false}, + {name => 'bPathCreate', optional => true, default => false}, + ); + + # Open the file + my $oFileIo = $self->driver()->openWrite($self->pathGet($xFileExp), + {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate, + bAtomic => $bAtomic}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIo', value => $oFileIo, trace => true}, + ); +} + +#################################################################################################################################### +# owner - change ownership of path/file +#################################################################################################################################### +sub owner +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathFileExp, + $strUser, + $strGroup + ) = + logDebugParam + ( + __PACKAGE__ . '->owner', \@_, + {name => 'strPathFileExp'}, + {name => 'strUser', required => false}, + {name => 'strGroup', required => false} + ); + + # Set ownership + $self->driver()->owner($self->pathGet($strPathFileExp), {strUser => $strUser, strGroup => $strGroup}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation + ); +} + +#################################################################################################################################### +# pathCreate - create path +#################################################################################################################################### +sub pathCreate +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + $strMode, + $bIgnoreExists, + $bCreateParent, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathCreate', \@_, + {name => 'strPathExp'}, + {name => 'strMode', optional => true, default => $self->{strDefaultPathMode}}, + {name => 'bIgnoreExists', optional => true, default => false}, + {name => 'bCreateParent', optional => true, default => false}, + ); + + # Create path + $self->driver()->pathCreate( + $self->pathGet($strPathExp), {strMode => $strMode, bIgnoreExists => $bIgnoreExists, bCreateParent => $bCreateParent}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation + ); +} + +#################################################################################################################################### +# pathExists - check if path exists +#################################################################################################################################### +sub pathExists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathExists', \@_, + {name => 'strPathExp'}, + ); + + # Check exists + my $bExists = $self->driver()->pathExists($self->pathGet($strPathExp)); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists} + ); +} + +#################################################################################################################################### +# pathGet - resolve a path expression into an absolute path +#################################################################################################################################### +sub pathGet +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, # File that that needs to be translated to a path + $bTemp, # Return the temp file name + ) = + logDebugParam + ( + __PACKAGE__ . '->pathGet', \@_, + {name => 'strPathExp', required => false, trace => true}, + {name => 'bTemp', optional => true, default => false, trace => true}, + ); + + # Path and file to be returned + my $strPath; + my $strFile; + + # Is this an absolute path type? + my $bAbsolute = false; + + if (defined($strPathExp) && index($strPathExp, qw(/)) == 0) + { + $bAbsolute = true; + $strPath = $strPathExp; + } + # Else it must be relative + else + { + $strPath = $self->pathBase(); + $strFile = $strPathExp; + } + + # Make sure a temp file is valid for this type and file + if ($bTemp) + { + # Error when temp files are not allowed + if (!$self->{bAllowTemp}) + { + confess &log(ASSERT, "temp file not supported for storage '" . $self->pathBase() . "'"); + } + + # The file must be defined + if (!$bAbsolute) + { + if (!defined($strFile)) + { + confess &log(ASSERT, 'file part must be defined when temp file specified'); + } + } + } + + # Combine path and file + $strPath .= defined($strFile) ? ($strPath =~ /\/$/ ? '' : qw{/}) . "${strFile}" : ''; + + # Add temp extension + $strPath .= $bTemp ? ".$self->{strTempExtension}" : ''; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strPath', value => $strPath, trace => true} + ); +} + +#################################################################################################################################### +# Sync path so newly added file entries are not lost +#################################################################################################################################### +sub pathSync +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathSync', \@_, + {name => 'strPathExp'}, + ); + + $self->driver()->pathSync($self->pathGet($strPathExp)); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# remove - remove path/file +#################################################################################################################################### +sub remove +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xstryPathFileExp, + $bIgnoreMissing, + $bRecurse, + ) = + logDebugParam + ( + __PACKAGE__ . '->remove', \@_, + {name => 'xstryPathFileExp'}, + {name => 'bIgnoreMissing', optional => true, default => true}, + {name => 'bRecurse', optional => true, default => false, trace => true}, + ); + + # Evaluate expressions for all files + my @stryPathFileExp; + + if (ref($xstryPathFileExp)) + { + foreach my $strPathFileExp (@{$xstryPathFileExp}) + { + push(@stryPathFileExp, $self->pathGet($strPathFileExp)); + } + } + + # Remove path(s)/file(s) + my $bRemoved = $self->driver()->remove( + ref($xstryPathFileExp) ? \@stryPathFileExp : $self->pathGet($xstryPathFileExp), + {bIgnoreMissing => $bIgnoreMissing, bRecurse => $bRecurse}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bRemoved', value => $bRemoved} + ); +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub pathBase {shift->{strPathBase}} +sub driver {shift->{oDriver}} +sub cipherType {undef} +sub cipherPassUser {undef} + +1; diff --git a/test/lib/pgBackRestTest/Common/StoragePosix.pm b/test/lib/pgBackRestTest/Common/StoragePosix.pm new file mode 100644 index 000000000..e208f5adf --- /dev/null +++ b/test/lib/pgBackRestTest/Common/StoragePosix.pm @@ -0,0 +1,982 @@ +#################################################################################################################################### +# Posix Storage +# +# Implements storage functions for Posix-compliant file systems. +#################################################################################################################################### +package pgBackRestTest::Common::StoragePosix; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use File::Basename qw(basename dirname); +use Fcntl qw(:mode); +use File::stat qw{lstat}; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Storage::Base; +use pgBackRestTest::Common::StoragePosixRead; +use pgBackRestTest::Common::StoragePosixWrite; + +#################################################################################################################################### +# Package name constant +#################################################################################################################################### +use constant STORAGE_POSIX_DRIVER => __PACKAGE__; + push @EXPORT, qw(STORAGE_POSIX_DRIVER); + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Create the class hash + my $self = {}; + bless $self, $class; + + # Assign function parameters, defaults, and log debug info + ( + my $strOperation, + $self->{bFileSync}, + $self->{bPathSync}, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'bFileSync', optional => true, default => true}, + {name => 'bPathSync', optional => true, default => true}, + ); + + # Set default temp extension + $self->{strTempExtension} = 'tmp'; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# exists - check if a path or file exists +#################################################################################################################################### +sub exists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->exists', \@_, + {name => 'strFile', trace => true}, + ); + + # Does the path/file exist? + my $bExists = true; + my $oStat = lstat($strFile); + + # Use stat to test if file exists + if (defined($oStat)) + { + # Check that it is actually a file + $bExists = !S_ISDIR($oStat->mode) ? true : false; + } + else + { + # If the error is not entry missing, then throw error + if (!$OS_ERROR{ENOENT}) + { + logErrorResult(ERROR_FILE_EXISTS, "unable to test if file '${strFile}' exists", $OS_ERROR); + } + + $bExists = false; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists, trace => true} + ); +} + +#################################################################################################################################### +# info - get information for path/file +#################################################################################################################################### +sub info +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathFile, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->info', \@_, + {name => 'strFile', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + # Stat the path/file + my $oInfo = lstat($strPathFile); + + # Check for errors + if (!defined($oInfo)) + { + if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) + { + logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to stat '${strPathFile}'", $OS_ERROR); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oInfo', value => $oInfo, trace => true} + ); +} + +#################################################################################################################################### +# linkCreate +#################################################################################################################################### +sub linkCreate +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strSourcePathFile, + $strDestinationLink, + $bHard, + $bPathCreate, + $bIgnoreExists, + ) = + logDebugParam + ( + __PACKAGE__ . '->linkCreate', \@_, + {name => 'strSourcePathFile', trace => true}, + {name => 'strDestinationLink', trace => true}, + {name => 'bHard', optional=> true, default => false, trace => true}, + {name => 'bPathCreate', optional=> true, default => true, trace => true}, + {name => 'bIgnoreExists', optional => true, default => false, trace => true}, + ); + + if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink))) + { + my $strMessage = "unable to create link '${strDestinationLink}'"; + + # If parent path or source is missing + if ($OS_ERROR{ENOENT}) + { + # Check if source is missing + if (!$self->exists($strSourcePathFile)) + { + confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING); + } + + if (!$bPathCreate) + { + confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING); + } + + # Create parent path + $self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true}); + + # Create link + $self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard}); + } + # Else if link already exists + elsif ($OS_ERROR{EEXIST}) + { + if (!$bIgnoreExists) + { + confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS); + } + } + else + { + logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR); + } + } + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# linkDestination - get destination of symlink +#################################################################################################################################### +sub linkDestination +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strLink, + ) = + logDebugParam + ( + __PACKAGE__ . '->linkDestination', \@_, + {name => 'strLink', trace => true}, + ); + + # Get link destination + my $strLinkDestination = readlink($strLink); + + # Check for errors + if (!defined($strLinkDestination)) + { + logErrorResult( + $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to get destination for link ${strLink}", $OS_ERROR); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strLinkDestination', value => $strLinkDestination, trace => true} + ); +} + +#################################################################################################################################### +# list - list all files/paths in path +#################################################################################################################################### +sub list +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->list', \@_, + {name => 'strPath', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + # Working variables + my @stryFileList; + my $hPath; + + # Attempt to open the path + if (opendir($hPath, $strPath)) + { + @stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath)); + close($hPath); + } + # Else process errors + else + { + # Ignore the error if the file is missing and missing files should be ignored + if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) + { + logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to read path '${strPath}'", $OS_ERROR); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true} + ); +} + +#################################################################################################################################### +# manifest - build path/file/link manifest starting with base path and including all subpaths +#################################################################################################################################### +sub manifest +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $bIgnoreMissing, + $strFilter, + ) = + logDebugParam + ( + __PACKAGE__ . '->manifest', \@_, + {name => 'strPath', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + {name => 'strFilter', optional => true, trace => true}, + ); + + # Generate the manifest + my $hManifest = {}; + $self->manifestRecurse($strPath, undef, 0, $hManifest, $bIgnoreMissing, $strFilter); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hManifest', value => $hManifest, trace => true} + ); +} + +sub manifestRecurse +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $strSubPath, + $iDepth, + $hManifest, + $bIgnoreMissing, + $strFilter, + ) = + logDebugParam + ( + __PACKAGE__ . '::manifestRecurse', \@_, + {name => 'strPath', trace => true}, + {name => 'strSubPath', required => false, trace => true}, + {name => 'iDepth', default => 0, trace => true}, + {name => 'hManifest', required => false, trace => true}, + {name => 'bIgnoreMissing', required => false, default => false, trace => true}, + {name => 'strFilter', required => false, trace => true}, + ); + + # Set operation and debug strings + my $strPathRead = $strPath . (defined($strSubPath) ? "/${strSubPath}" : ''); + my $hPath; + + # If this is the top level stat the path to discover if it is actually a file + my $oPathInfo = $self->info($strPathRead, {bIgnoreMissing => $bIgnoreMissing}); + + if (defined($oPathInfo)) + { + # If the initial path passed is a file then generate the manifest for just that file + if ($iDepth == 0 && !S_ISDIR($oPathInfo->mode())) + { + $hManifest->{basename($strPathRead)} = $self->manifestStat($strPathRead); + } + # Else read as a normal directory + else + { + # Get a list of all files in the path (including .) + my @stryFileList = @{$self->list($strPathRead, {bIgnoreMissing => $iDepth != 0})}; + unshift(@stryFileList, '.'); + my $hFileStat = $self->manifestList($strPathRead, \@stryFileList, $strFilter); + + # Loop through all subpaths/files in the path + foreach my $strFile (keys(%{$hFileStat})) + { + my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : "/${strFile}")); + $hManifest->{$strManifestFile} = $hFileStat->{$strFile}; + + # Recurse into directories + if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.)) + { + $self->manifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest); + } + } + } + } + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +sub manifestList +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $stryFile, + $strFilter, + ) = + logDebugParam + ( + __PACKAGE__ . '->manifestList', \@_, + {name => 'strPath', trace => true}, + {name => 'stryFile', trace => true}, + {name => 'strFilter', required => false, trace => true}, + ); + + my $hFileStat = {}; + + foreach my $strFile (@{$stryFile}) + { + if ($strFile ne '.' && defined($strFilter) && $strFilter ne $strFile) + { + next; + } + + $hFileStat->{$strFile} = $self->manifestStat("${strPath}" . ($strFile eq qw(.) ? '' : "/${strFile}")); + + if (!defined($hFileStat->{$strFile})) + { + delete($hFileStat->{$strFile}); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hFileStat', value => $hFileStat, trace => true} + ); +} + +sub manifestStat +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->manifestStat', \@_, + {name => 'strFile', trace => true}, + ); + + # Stat the path/file, ignoring any that are missing + my $oStat = $self->info($strFile, {bIgnoreMissing => true}); + + # Generate file data if stat succeeded (i.e. file exists) + my $hFile; + + if (defined($oStat)) + { + # Check for regular file + if (S_ISREG($oStat->mode)) + { + $hFile->{type} = 'f'; + + # Get size + $hFile->{size} = $oStat->size; + + # Get modification time + $hFile->{modification_time} = $oStat->mtime; + } + # Check for directory + elsif (S_ISDIR($oStat->mode)) + { + $hFile->{type} = 'd'; + } + # Check for link + elsif (S_ISLNK($oStat->mode)) + { + $hFile->{type} = 'l'; + $hFile->{link_destination} = $self->linkDestination($strFile); + } + # Not a recognized type + else + { + confess &log(ERROR, "${strFile} is not of type directory, file, or link", ERROR_FILE_INVALID); + } + + # Get user name + $hFile->{user} = getpwuid($oStat->uid); + + # Get group name + $hFile->{group} = getgrgid($oStat->gid); + + # Get mode + if ($hFile->{type} ne 'l') + { + $hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode)); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hFile', value => $hFile, trace => true} + ); +} + +#################################################################################################################################### +# move - move path/file +#################################################################################################################################### +sub move +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strSourceFile, + $strDestinationFile, + $bPathCreate, + ) = + logDebugParam + ( + __PACKAGE__ . '->move', \@_, + {name => 'strSourceFile', trace => true}, + {name => 'strDestinationFile', trace => true}, + {name => 'bPathCreate', default => false, trace => true}, + ); + + # Get source and destination paths + my $strSourcePathFile = dirname($strSourceFile); + my $strDestinationPathFile = dirname($strDestinationFile); + + # Move the file + if (!rename($strSourceFile, $strDestinationFile)) + { + my $strMessage = "unable to move '${strSourceFile}'"; + + # If something is missing determine if it is the source or destination + if ($OS_ERROR{ENOENT}) + { + if (!$self->exists($strSourceFile)) + { + logErrorResult(ERROR_FILE_MISSING, "${strMessage} because it is missing"); + } + + if ($bPathCreate) + { + # Attempt to create the path - ignore exists here in case another process creates it first + $self->pathCreate($strDestinationPathFile, {bCreateParent => true, bIgnoreExists => true}); + + # Try move again + $self->move($strSourceFile, $strDestinationFile); + } + else + { + logErrorResult(ERROR_PATH_MISSING, "${strMessage} to missing path '${strDestinationPathFile}'"); + } + } + # Else raise the error + else + { + logErrorResult(ERROR_FILE_MOVE, "${strMessage} to '${strDestinationFile}'", $OS_ERROR); + } + } + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# openRead - open file for reading +#################################################################################################################################### +sub openRead +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->openRead', \@_, + {name => 'strFile', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + my $oFileIO = new pgBackRestTest::Common::StoragePosixRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIO', value => $oFileIO, trace => true}, + ); +} + +#################################################################################################################################### +# openWrite - open file for writing +#################################################################################################################################### +sub openWrite +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + $strMode, + $strUser, + $strGroup, + $lTimestamp, + $bPathCreate, + $bAtomic, + ) = + logDebugParam + ( + __PACKAGE__ . '->openWrite', \@_, + {name => 'strFile', trace => true}, + {name => 'strMode', optional => true, trace => true}, + {name => 'strUser', optional => true, trace => true}, + {name => 'strGroup', optional => true, trace => true}, + {name => 'lTimestamp', optional => true, trace => true}, + {name => 'bPathCreate', optional => true, trace => true}, + {name => 'bAtomic', optional => true, trace => true}, + ); + + my $oFileIO = new pgBackRestTest::Common::StoragePosixWrite( + $self, $strFile, + {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate, + bAtomic => $bAtomic, bSync => $self->{bFileSync}}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIO', value => $oFileIO, trace => true}, + ); +} + +#################################################################################################################################### +# owner - change ownership of path/file +#################################################################################################################################### +sub owner +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFilePath, + $strUser, + $strGroup, + ) = + logDebugParam + ( + __PACKAGE__ . '->owner', \@_, + {name => 'strFilePath', trace => true}, + {name => 'strUser', optional => true, trace => true}, + {name => 'strGroup', optional => true, trace => true}, + ); + + # Only proceed if user or group was specified + if (defined($strUser) || defined($strGroup)) + { + my $strMessage = "unable to set ownership for '${strFilePath}'"; + my $iUserId; + my $iGroupId; + + # If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that + # both user and group be set. + my $oStat = $self->info($strFilePath); + + if (!defined($strUser)) + { + $iUserId = $oStat->uid; + } + + if (!defined($strGroup)) + { + $iGroupId = $oStat->gid; + } + + # Lookup user if specified + if (defined($strUser)) + { + $iUserId = getpwnam($strUser); + + if (!defined($iUserId)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist"); + } + } + + # Lookup group if specified + if (defined($strGroup)) + { + $iGroupId = getgrnam($strGroup); + + if (!defined($iGroupId)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist"); + } + } + + # Set ownership on the file if the user or group would be changed + if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid) + { + if (!chown($iUserId, $iGroupId, $strFilePath)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR); + } + } + } + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# pathCreate - create path +#################################################################################################################################### +sub pathCreate +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $strMode, + $bIgnoreExists, + $bCreateParent, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathCreate', \@_, + {name => 'strPath', trace => true}, + {name => 'strMode', optional => true, default => '0750', trace => true}, + {name => 'bIgnoreExists', optional => true, default => false, trace => true}, + {name => 'bCreateParent', optional => true, default => false, trace => true}, + ); + + # Attempt to create the directory + if (!mkdir($strPath, oct($strMode))) + { + my $strMessage = "unable to create path '${strPath}'"; + + # If parent path is missing + if ($OS_ERROR{ENOENT}) + { + if (!$bCreateParent) + { + confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING); + } + + # Create parent path + $self->pathCreate(dirname($strPath), {strMode => $strMode, bIgnoreExists => true, bCreateParent => $bCreateParent}); + + # Create path + $self->pathCreate($strPath, {strMode => $strMode, bIgnoreExists => true}); + } + # Else if path already exists + elsif ($OS_ERROR{EEXIST}) + { + if (!$bIgnoreExists) + { + confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS); + } + } + else + { + logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR); + } + } + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# pathExists - check if path exists +#################################################################################################################################### +sub pathExists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathExists', \@_, + {name => 'strPath', trace => true}, + ); + + # Does the path/file exist? + my $bExists = true; + my $oStat = lstat($strPath); + + # Use stat to test if path exists + if (defined($oStat)) + { + # Check that it is actually a path + $bExists = S_ISDIR($oStat->mode) ? true : false; + } + else + { + # If the error is not entry missing, then throw error + if (!$OS_ERROR{ENOENT}) + { + logErrorResult(ERROR_FILE_EXISTS, "unable to test if path '${strPath}' exists", $OS_ERROR); + } + + $bExists = false; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists, trace => true} + ); +} + +#################################################################################################################################### +# pathSync - perform fsync on path +#################################################################################################################################### +sub pathSync +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathSync', \@_, + {name => 'strPath', trace => true}, + ); + + open(my $hPath, "<", $strPath) + or confess &log(ERROR, "unable to open '${strPath}' for sync", ERROR_PATH_OPEN); + open(my $hPathDup, ">&", $hPath) + or confess &log(ERROR, "unable to duplicate '${strPath}' handle for sync", ERROR_PATH_OPEN); + + $hPathDup->sync() + or confess &log(ERROR, "unable to sync path '${strPath}'", ERROR_PATH_SYNC); + + close($hPathDup); + close($hPath); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# remove - remove path/file +#################################################################################################################################### +sub remove +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xstryPathFile, + $bIgnoreMissing, + $bRecurse, + ) = + logDebugParam + ( + __PACKAGE__ . '->remove', \@_, + {name => 'xstryPathFile', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + {name => 'bRecurse', optional => true, default => false, trace => true}, + ); + + # Working variables + my $bRemoved = true; + + # Remove a tree + if ($bRecurse) + { + my $oManifest = $self->manifest($xstryPathFile, {bIgnoreMissing => true}); + + # Iterate all files in the manifest + foreach my $strFile (sort({$b cmp $a} keys(%{$oManifest}))) + { + # remove directory + if ($oManifest->{$strFile}{type} eq 'd') + { + my $xstryPathFileRemove = $strFile eq '.' ? $xstryPathFile : "${xstryPathFile}/${strFile}"; + + if (!rmdir($xstryPathFileRemove)) + { + # Throw error if this is not an ignored missing path + if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) + { + logErrorResult(ERROR_PATH_REMOVE, "unable to remove path '${strFile}'", $OS_ERROR); + } + } + } + # Remove file + else + { + $self->remove("${xstryPathFile}/${strFile}", {bIgnoreMissing => true}); + } + } + } + # Only remove the specified file + else + { + foreach my $strFile (ref($xstryPathFile) ? @{$xstryPathFile} : ($xstryPathFile)) + { + if (unlink($strFile) != 1) + { + $bRemoved = false; + + # Throw error if this is not an ignored missing file + if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) + { + logErrorResult( + $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to remove file '${strFile}'", $OS_ERROR); + } + } + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bRemoved', value => $bRemoved, trace => true} + ); +} + +#################################################################################################################################### +# Getters/Setters +#################################################################################################################################### +sub className {STORAGE_POSIX_DRIVER} +sub tempExtension {shift->{strTempExtension}} +sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift} + +1; diff --git a/test/lib/pgBackRestTest/Common/StoragePosixRead.pm b/test/lib/pgBackRestTest/Common/StoragePosixRead.pm new file mode 100644 index 000000000..208b9e7bf --- /dev/null +++ b/test/lib/pgBackRestTest/Common/StoragePosixRead.pm @@ -0,0 +1,105 @@ +#################################################################################################################################### +# Posix File Read +#################################################################################################################################### +package pgBackRestTest::Common::StoragePosixRead; +use parent 'pgBackRest::Common::Io::Handle'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Fcntl qw(O_RDONLY); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oDriver, + $strName, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oDriver', trace => true}, + {name => 'strName', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + # Open the file + my $fhFile; + + if (!sysopen($fhFile, $strName, O_RDONLY)) + { + if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) + { + logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to open '${strName}'", $OS_ERROR); + } + + undef($fhFile); + } + + # Create IO object if open succeeded + my $self; + + if (defined($fhFile)) + { + # Set file mode to binary + binmode($fhFile); + + # Create the class hash + $self = $class->SUPER::new("'${strName}'", $fhFile); + bless $self, $class; + + # Set variables + $self->{oDriver} = $oDriver; + $self->{strName} = $strName; + $self->{fhFile} = $fhFile; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# close - close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + if (defined($self->handle())) + { + # Close the file + close($self->handle()); + undef($self->{fhFile}); + + # Close parent + $self->SUPER::close(); + } + + return true; +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub handle {shift->{fhFile}} +sub name {shift->{strName}} + +1; diff --git a/test/lib/pgBackRestTest/Common/StoragePosixWrite.pm b/test/lib/pgBackRestTest/Common/StoragePosixWrite.pm new file mode 100644 index 000000000..8b3fbbf30 --- /dev/null +++ b/test/lib/pgBackRestTest/Common/StoragePosixWrite.pm @@ -0,0 +1,209 @@ +#################################################################################################################################### +# Posix File Write +#################################################################################################################################### +package pgBackRestTest::Common::StoragePosixWrite; +use parent 'pgBackRest::Common::Io::Handle'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); +use File::Basename qw(dirname); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; + +use pgBackRest::Common::Io::Handle; +use pgBackRest::Storage::Base; + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oDriver, + $strName, + $strMode, + $strUser, + $strGroup, + $lTimestamp, + $bPathCreate, + $bAtomic, + $bSync, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oDriver', trace => true}, + {name => 'strName', trace => true}, + {name => 'strMode', optional => true, trace => true}, + {name => 'strUser', optional => true, trace => true}, + {name => 'strGroup', optional => true, trace => true}, + {name => 'lTimestamp', optional => true, trace => true}, + {name => 'bPathCreate', optional => true, default => false, trace => true}, + {name => 'bAtomic', optional => true, default => false, trace => true}, + {name => 'bSync', optional => true, default => true, trace => true}, + ); + + # Create the class hash + my $self = $class->SUPER::new("'${strName}'"); + bless $self, $class; + + # Set variables + $self->{oDriver} = $oDriver; + $self->{strName} = $strName; + $self->{strMode} = $strMode; + $self->{strUser} = $strUser; + $self->{strGroup} = $strGroup; + $self->{lTimestamp} = $lTimestamp; + $self->{bPathCreate} = $bPathCreate; + $self->{bAtomic} = $bAtomic; + $self->{bSync} = $bSync; + + # If atomic create temp filename + if ($self->{bAtomic}) + { + # Create temp file name + $self->{strNameTmp} = "$self->{strName}." . $self->{oDriver}->tempExtension(); + } + + # Open file on first write to avoid creating extraneous files on error + $self->{bOpened} = false; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# open - open the file +#################################################################################################################################### +sub open +{ + my $self = shift; + + # Get the file name + my $strFile = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName}; + + # Open the file + if (!sysopen( + $self->{fhFile}, $strFile, O_WRONLY | O_CREAT | O_TRUNC, oct(defined($self->{strMode}) ? $self->{strMode} : '0666'))) + { + # If the path does not exist create it if requested + if ($OS_ERROR{ENOENT} && $self->{bPathCreate}) + { + $self->{oDriver}->pathCreate(dirname($strFile), {bIgnoreExists => true, bCreateParent => true}); + $self->{bPathCreate} = false; + return $self->open(); + } + + logErrorResult($OS_ERROR{ENOENT} ? ERROR_PATH_MISSING : ERROR_FILE_OPEN, "unable to open '${strFile}'", $OS_ERROR); + } + + # Set file mode to binary + binmode($self->{fhFile}); + + # Set the owner + $self->{oDriver}->owner($strFile, {strUser => $self->{strUser}, strGroup => $self->{strGroup}}); + + # Set handle + $self->handleWriteSet($self->{fhFile}); + + # Mark file as opened + $self->{bOpened} = true; + + return true; +} + +#################################################################################################################################### +# write - write data to a file +#################################################################################################################################### +sub write +{ + my $self = shift; + my $rtBuffer = shift; + + # Open file if it is not open already + $self->open() if !$self->opened(); + + return $self->SUPER::write($rtBuffer); +} + +#################################################################################################################################### +# close - close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + if (defined($self->handle())) + { + # Sync the file + if ($self->{bSync}) + { + $self->handle()->sync(); + } + + # Close the file + close($self->handle()); + undef($self->{fhFile}); + + # Get current filename + my $strCurrentName = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName}; + + # Set the modification time + if (defined($self->{lTimestamp})) + { + utime(time(), $self->{lTimestamp}, $strCurrentName) + or logErrorResult(ERROR_FILE_WRITE, "unable to set time for '${strCurrentName}'", $OS_ERROR); + } + + # Move the file from temp to final if atomic + if ($self->{bAtomic}) + { + $self->{oDriver}->move($strCurrentName, $self->{strName}); + } + + # Set result + $self->resultSet(COMMON_IO_HANDLE, $self->{lSize}); + + # Close parent + $self->SUPER::close(); + } + + return true; +} + +#################################################################################################################################### +# Close the handle if it is open (in case close() was never called) +#################################################################################################################################### +sub DESTROY +{ + my $self = shift; + + if (defined($self->handle())) + { + CORE::close($self->handle()); + undef($self->{fhFile}); + } +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub handle {shift->{fhFile}} +sub opened {shift->{bOpened}} +sub name {shift->{strName}} + +1; diff --git a/test/test.pl b/test/test.pl index a02b36d42..dc3cb8564 100755 --- a/test/test.pl +++ b/test/test.pl @@ -31,8 +31,6 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Wait; -use pgBackRest::Storage::Posix::Driver; -use pgBackRest::Storage::Local; use pgBackRest::Version; use pgBackRestBuild::Build; @@ -55,6 +53,8 @@ use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::JobTest; use pgBackRestTest::Common::ListTest; use pgBackRestTest::Common::RunTest; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; use pgBackRestTest::Common::VmTest; #################################################################################################################################### @@ -328,8 +328,8 @@ eval $strTestPath = cwd() . '/test'; } - my $oStorageTest = new pgBackRest::Storage::Local( - $strTestPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageTest = new pgBackRestTest::Common::Storage( + $strTestPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); if ($bCoverageOnly) { @@ -357,8 +357,8 @@ eval # Get the base backrest path my $strBackRestBase = dirname(dirname(abs_path($0))); - my $oStorageBackRest = new pgBackRest::Storage::Local( - $strBackRestBase, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorageBackRest = new pgBackRestTest::Common::Storage( + $strBackRestBase, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); ################################################################################################################################ # Build Docker containers