db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

Storage and IO layer refactor:

Browse Source 
Refactor storage layer to allow for new repository filesystems using drivers. (Reviewed by Cynthia Shang.)
Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters. (Reviewed by Cynthia Shang.)
This commit is contained in:
David Steele
2017-06-09 17:51:41 -04:00
parent 7e982f05f5
commit de7fc37f88
183 changed files with 17880 additions and 14734 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
doc/lib/BackRestDoc/Common/Doc.pm
View File

@@ -14,7 +14,6 @@ use XML::Checker::Parser;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
####################################################################################################################################
# CONSTRUCTOR
@@ -33,53 +32,44 @@ sub new
(
my $strOperation,
$self->{strFileName},
my $bCached
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strFileName', required => false},
{name => 'bCached', default => false}
);
# Load the doc from a file if one has been defined
if (defined($self->{strFileName}))
{
if ($bCached)
my $oParser = XML::Checker::Parser->new(ErrorContext => 2, Style => 'Tree');
$oParser->set_sgml_search_path(dirname(dirname($0)) . '/doc/xml/dtd');
my $oTree;
eval
{
$self->oDoc = XMLin(fileStringRead($self->{strFileName}));
}
else
{
my $oParser = XML::Checker::Parser->new(ErrorContext => 2, Style => 'Tree');
$oParser->set_sgml_search_path(dirname(dirname($0)) . '/doc/xml/dtd');
my $oTree;
eval
local $XML::Checker::FAIL = sub
{
local $XML::Checker::FAIL = sub
{
my $iCode = shift;
my $iCode = shift;
die XML::Checker::error_string($iCode, @_);
};
$oTree = $oParser->parsefile($self->{strFileName});
return true;
}
# Report any error that stopped parsing
or do
{
my $strException = $EVAL_ERROR;
$strException =~ s/at \/.*?$//s; # remove module line number
die "malformed xml in '$self->{strFileName}':\n" . trim($strException);
die XML::Checker::error_string($iCode, @_);
};
# Parse and build the doc
$self->{oDoc} = $self->build($self->parse(${$oTree}[0], ${$oTree}[1]));
$oTree = $oParser->parsefile($self->{strFileName});
return true;
}
# Report any error that stopped parsing
or do
{
my $strException = $EVAL_ERROR;
$strException =~ s/at \/.*?$//s; # remove module line number
die "malformed xml in '$self->{strFileName}':\n" . trim($strException);
};
# Parse and build the doc
$self->{oDoc} = $self->build($self->parse(${$oTree}[0], ${$oTree}[1]));
}
# Else create a blank doc
else

3
doc/lib/BackRestDoc/Common/DocConfig.pm
View File

@@ -15,7 +15,6 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::Config::ConfigHelp;
use pgBackRest::FileCommon;
use pgBackRest::Version;
####################################################################################################################################
@@ -410,7 +409,7 @@ sub helpDataWrite
"1;\n";
# Write the perl module into the lib path
fileStringWrite(dirname(dirname($0)) . '/lib/pgBackRest/Config/ConfigHelpData.pm', $strHelpData, false);
$oManifest->storage()->put(dirname(dirname($0)) . '/lib/pgBackRest/Config/ConfigHelpData.pm', $strHelpData);
# Return from function and log return values if any
logDebugReturn($strOperation);

12
doc/lib/BackRestDoc/Common/DocExecute.pm
View File

@@ -18,7 +18,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -562,7 +561,7 @@ sub backrestConfig
my $strLocalFile = '/home/' . DOC_USER . '/data/pgbackrest.conf';
# Save the ini file
fileStringWrite($strLocalFile, iniRender($self->{config}{$strHostName}{$$hCacheKey{file}}, true));
$self->{oManifest}->storage()->put($strLocalFile, iniRender($self->{config}{$strHostName}{$$hCacheKey{file}}, true));
$oHost->copyTo(
$strLocalFile, $$hCacheKey{file},
@@ -579,10 +578,10 @@ sub backrestConfig
delete($$oConfigClean{&CONFIG_SECTION_GLOBAL});
}
fileStringWrite("${strLocalFile}.clean", iniRender($oConfigClean, true));
$self->{oManifest}->storage()->put("${strLocalFile}.clean", iniRender($oConfigClean, true));
# Push config file into the cache
$strConfig = fileStringRead("${strLocalFile}.clean");
$strConfig = ${$self->{oManifest}->storage()->get("${strLocalFile}.clean")};
my @stryConfig = undef;
@@ -662,7 +661,8 @@ sub postgresConfig
if (!defined(${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base}) && $self->{bExe})
{
${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} = fileStringRead($strLocalFile);
${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} =
${$self->{oManifest}->storage()->get($strLocalFile)};
}
my $oConfigHash = $self->{'pg-config'}{$strHostName}{$$hCacheKey{file}};
@@ -712,7 +712,7 @@ sub postgresConfig
# Save the conf file
if ($self->{bExe})
{
fileStringWrite($strLocalFile, $$oConfigHash{base} .
$self->{oManifest}->storage()->put($strLocalFile, $$oConfigHash{base} .
(defined($strConfig) ? "\n# pgBackRest Configuration\n${strConfig}\n" : ''));
$oHost->copyTo($strLocalFile, $$hCacheKey{file}, 'postgres:postgres', '640');

16
doc/lib/BackRestDoc/Common/DocManifest.pm
View File

@@ -15,7 +15,6 @@ use JSON::PP;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
####################################################################################################################################
# File constants
@@ -56,6 +55,7 @@ sub new
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorage},
$self->{stryKeyword},
$self->{stryRequire},
$self->{stryInclude},
@@ -68,6 +68,7 @@ sub new
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorage'},
{name => 'stryKeyword'},
{name => 'stryRequire'},
{name => 'stryInclude'},
@@ -627,16 +628,16 @@ sub cacheRead
my $strCacheFile = $self->{bDeploy} ? $self->{strExeCacheDeploy} : $self->{strExeCacheLocal};
if (!fileExists($strCacheFile) && !$self->{bDeploy})
if (!$self->storage()->exists($strCacheFile) && !$self->{bDeploy})
{
$strCacheFile = $self->{strExeCacheDeploy};
}
if (fileExists($strCacheFile))
if ($self->storage()->exists($strCacheFile))
{
my ($strKeyword, $strRequire) = $self->cacheKey();
my $oJSON = JSON::PP->new()->allow_nonref();
$self->{hCache} = $oJSON->decode(fileStringRead($strCacheFile));
$self->{hCache} = $oJSON->decode(${$self->storage()->get($strCacheFile)});
foreach my $strSource (sort(keys(%{${$self->{oManifest}}{source}})))
{
@@ -681,7 +682,7 @@ sub cacheWrite
if (defined($self->{hCache}))
{
my $oJSON = JSON::PP->new()->canonical()->allow_nonref()->pretty();
fileStringWrite($strCacheFile, $oJSON->encode($self->{hCache}), false);
$self->storage()->put($strCacheFile, $oJSON->encode($self->{hCache}));
}
# Return from function and log return values if any
@@ -719,4 +720,9 @@ sub cacheReset
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub storage {shift->{oStorage}};
1;

1
doc/lib/BackRestDoc/Custom/DocCustomRelease.pm
View File

@@ -14,7 +14,6 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::Config::ConfigHelp;
use pgBackRest::FileCommon;
use BackRestDoc::Common::DocRender;

11
doc/lib/BackRestDoc/Html/DocHtmlSite.pm
View File

@@ -19,7 +19,6 @@ use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -138,8 +137,8 @@ sub process
{
$strHtml = $self->{oManifest}->variableReplace(
new BackRestDoc::Html::DocHtmlPage(
$self->{oManifest}, $strPageId, $bMenu, $self->{bExe}, $bCompact, fileStringRead($self->{strCssFile}),
$bPretty)->process());
$self->{oManifest}, $strPageId, $bMenu, $self->{bExe}, $bCompact,
${$self->{oManifest}->storage()->get($self->{strCssFile})}, $bPretty)->process());
return true;
}
@@ -154,8 +153,8 @@ sub process
$strHtml = $self->{oManifest}->variableReplace(
new BackRestDoc::Html::DocHtmlPage(
$self->{oManifest}, $strPageId, $bMenu, $self->{bExe}, $bCompact, fileStringRead($self->{strCssFile}),
$bPretty)->process());
$self->{oManifest}, $strPageId, $bMenu, $self->{bExe}, $bCompact,
${$self->{oManifest}->storage()->get($self->{strCssFile})}, $bPretty)->process());
}
else
{
@@ -165,7 +164,7 @@ sub process
# Save the html page
my $strFile = "$self->{strHtmlPath}/" . (defined($$oRenderOut{file}) ? $$oRenderOut{file} : "${strPageId}.html");
fileStringWrite($strFile, $strHtml, false);
$self->{oManifest}->storage()->put($strFile, $strHtml);
}
# Return from function and log return values if any

8
doc/lib/BackRestDoc/Latex/DocLatex.pm
View File

@@ -19,7 +19,6 @@ use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -98,9 +97,10 @@ sub process
copy("$self->{oManifest}{strDocPath}/resource/latex/cds-logo.eps", "$self->{strLatexPath}/logo.eps")
or confess &log(ERROR, "unable to copy logo");
my $strLatex = $self->{oManifest}->variableReplace(fileStringRead($self->{strPreambleFile}), 'latex') . "\n";
my $strLatex = $self->{oManifest}->variableReplace(
${$self->{oManifest}->storage()->get($self->{strPreambleFile})}, 'latex') . "\n";
# !!! Temp hack for underscores in filename
# ??? Temp hack for underscores in filename
$strLatex =~ s/pgaudit\\\_doc/pgaudit\_doc/g;
foreach my $strPageId ($self->{oManifest}->renderOutList(RENDER_TYPE_PDF))
@@ -143,7 +143,7 @@ sub process
my $strLatexFileName = $self->{oManifest}->variableReplace("$self->{strLatexPath}/" . $$oRender{file} . '.tex');
fileStringWrite($strLatexFileName, $strLatex, false);
$self->{oManifest}->storage()->put($strLatexFileName, $strLatex);
executeTest("pdflatex -output-directory=$self->{strLatexPath} -shell-escape $strLatexFileName",
{bSuppressStdErr => true});

1
doc/lib/BackRestDoc/Latex/DocLatexSection.pm
View File

@@ -15,7 +15,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::ConfigHelp;
use pgBackRest::FileCommon;
use BackRestDoc::Common::DocManifest;

8
doc/lib/BackRestDoc/Markdown/DocMarkdown.pm
View File

@@ -17,7 +17,6 @@ use Storable qw(dclone);
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -96,10 +95,9 @@ sub process
&log(INFO, " render out: ${strRenderOutId}");
# Save the html page
fileStringWrite($strFile,
$self->{oManifest}->variableReplace((new BackRestDoc::Markdown::DocMarkdownRender($self->{oManifest},
$strRenderOutId, $self->{bExe}))->process()),
false);
$self->{oManifest}->storage()->put(
$strFile, $self->{oManifest}->variableReplace((new BackRestDoc::Markdown::DocMarkdownRender($self->{oManifest},
$strRenderOutId, $self->{bExe}))->process()));
}
# Return from function and log return values if any