db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-04-23 11:58:50 +02:00
Code Issues Releases Activity

Improve code generation performance.

Browse Source
This commit is contained in:
David Steele 2018-03-09 23:00:20 -05:00
parent b987f05f35
commit f047cd0dfd
5 changed files with 114 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
build/lib/pgBackRestBuild/Build.pm
View File

@ -162,25 +162,8 @@ sub buildAll
$strExt = $strFileType eq BLD_C ? $strFileExt : "${strFileExt}h";
}
# Only save the file if the content has changed
my $strFileName = "${strPath}/${strFile}.auto.${strExt}";
my $strContent = trim($rhSource->{$strFileType}) . "\n";
my $bChanged = true;
if ($oStorage->exists($strFileName))
{
my $strOldContent = ${$oStorage->get($strFileName)};
if ($strContent eq $strOldContent)
{
$bChanged = false;
}
}
if ($bChanged)
{
$oStorage->put($strFileName, $strContent);
}
# Save the file
$oStorage->put("${strPath}/${strFile}.auto.${strExt}", trim($rhSource->{$strFileType}) . "\n");
}
}
}

2
doc/xml/release.xml
View File

@ -110,7 +110,7 @@
<release-development-list>
<release-item>
<p>Improve bin and libc build performance.</p>
<p>Buld performance improvements. Improve bin and libc build performance. Improve code generation performance.</p>
</release-item>
<release-item>

20
libc/build/lib/pgBackRestLibC/Build.pm
View File

@ -297,24 +297,8 @@ sub buildXsAll
"\n" .
"1;\n";
# Only save the file if it has changed
my $strLibFile = 'lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm';
my $bChanged = true;
if ($oStorage->exists($strLibFile))
{
my $strContentOld = ${$oStorage->get($strLibFile)};
if ($strContent eq $strContentOld)
{
$bChanged = false;
}
}
if ($bChanged)
{
$oStorage->put($strLibFile, $strContent);
}
# Save the file
$oStorage->put('lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm', $strContent);
# Build error file
#-------------------------------------------------------------------------------------------------------------------------------

57
test/lib/pgBackRestTest/Common/BuildTest.pm Normal file
View File

@ -0,0 +1,57 @@
####################################################################################################################################
# Build Binaries and Auto-Generate Code
####################################################################################################################################
package pgBackRestTest::Common::BuildTest;
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
# use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::Log;
####################################################################################################################################
# Find last modification time in a list of directories, with optional filters
####################################################################################################################################
sub buildLastModTime
{
my $oStorage = shift;
my $strBasePath = shift;
my $rstrySubPath = shift;
my $strPattern = shift;
my $lTimestampLast = 0;
foreach my $strSubPath (defined($rstrySubPath) ? @{$rstrySubPath} : (''))
{
my $hManifest = $oStorage->manifest($strBasePath . ($strSubPath eq '' ? '' : "/${strSubPath}"));
foreach my $strFile (sort(keys(%{$hManifest})))
{
next if (defined($strPattern) && $strFile !~ /$strPattern/);
if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
{
$lTimestampLast = $hManifest->{$strFile}{modification_time};
}
}
}
if ($lTimestampLast == 0)
{
confess &log(ERROR, "no files found");
}
return $lTimestampLast;
}
push @EXPORT, qw(buildLastModTime);
1;

59
test/test.pl
View File

@ -47,6 +47,7 @@ use pgBackRestBuild::Error::Data;
use BackRestDoc::Custom::DocCustomRelease;
use pgBackRestTest::Common::BuildTest;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::CiTest;
use pgBackRestTest::Common::DefineTest;
@ -238,6 +239,7 @@ eval
}
logLevelSet(uc($strLogLevel), uc($strLogLevel), OFF);
&log(INFO, "test begin - log level ${strLogLevel}");
if (@stryModuleTest != 0 && @stryModule != 1)
{
@ -303,9 +305,15 @@ eval
{
# Auto-generate C files
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, "autogenerate C code");
&log(INFO, "check code autogenerate");
errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")});
if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['build']) >
buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['src'], '\.auto\.c$'))
{
&log(INFO, " autogenerate C code");
my $rhBuild =
{
'config' =>
@ -334,16 +342,21 @@ eval
};
buildAll("${strBackRestBase}/src", $rhBuild);
}
# Auto-generate XS files
#
# Use statements are put here so this will be easy to get rid of someday.
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, "autogenerate Perl code");
if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc/build']) >
buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc', 'lib'], '(\.auto\.pm|Auto\.pm)$'))
{
&log(INFO, " autogenerate Perl code");
use lib dirname(dirname($0)) . '/libc/build/lib';
use pgBackRestLibC::Build; ## no critic (Modules::ProhibitConditionalUseStatements)
buildXsAll("${strBackRestBase}/libc");
}
if ($bGenOnly)
{
@ -351,6 +364,7 @@ eval
}
# Sync time to prevent build failures when running on VirtualBox.
&log(INFO, "sync vbox time");
my $strVBoxService = '/usr/sbin/VBoxService';
if ($oStorageTest->exists($strVBoxService))
@ -365,6 +379,8 @@ eval
(new pgBackRestTest::Common::CiTest($oStorageBackRest))->process();
}
&log(INFO, "check version info");
# Load the doc module dynamically since it is not supported on all systems
require BackRestDoc::Common::Doc;
BackRestDoc::Common::Doc->import();
@ -421,6 +437,7 @@ eval
if (!$bDryRun || $bVmOut)
{
&log(INFO, "cleanup old data and containers");
containerRemove('test-([0-9]+|build)');
for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++)
@ -464,6 +481,8 @@ eval
#---------------------------------------------------------------------------------------------------------------------------
if (!$bDryRun && $bBuildRequired)
{
&log(INFO, "check bin builds");
my $oVm = vmGet();
my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
@ -475,27 +494,14 @@ eval
my @stryBinSrcPath = ('src');
# Find the lastest modified time for dirs that affect the bin build
my $lTimestampLast = $oStorageBackRest->exists($strBinSmart) ? $oStorageBackRest->info($strBinSmart)->mtime : 0;
foreach my $strBinSrcPath (@stryBinSrcPath)
{
my $hManifest = $oStorageBackRest->manifest($strBinSrcPath);
foreach my $strFile (sort(keys(%{$hManifest})))
{
if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
{
$lTimestampLast = $hManifest->{$strFile}{modification_time};
}
}
}
my $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryBinSrcPath);
# Rebuild if the modification time of the smart file does equal the last changes in source paths
if ($bSmart)
{
if (!$oStorageBackRest->exists($strBinSmart) || $oStorageBackRest->info($strBinSmart)->mtime < $lTimestampLast)
{
&log(INFO, 'bin dependencies have changed, rebuilding...');
&log(INFO, ' bin dependencies have changed, rebuilding...');
$bRebuild = true;
}
@ -515,7 +521,7 @@ eval
if ($bRebuild)
{
&log(INFO, "build bin for ${strBuildVM} (${strBuildPath})");
&log(INFO, " build bin for ${strBuildVM} (${strBuildPath})");
executeTest(
"docker run -itd -h test-build --name=test-build" .
@ -554,25 +560,14 @@ eval
my @stryLibCSrcPath = ('libc', 'src');
# Find the lastest modified time for dirs that affect the libc build
foreach my $strLibCSrcPath (@stryLibCSrcPath)
{
my $hManifest = $oStorageBackRest->manifest($strLibCSrcPath);
foreach my $strFile (sort(keys(%{$hManifest})))
{
if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
{
$lTimestampLast = $hManifest->{$strFile}{modification_time};
}
}
}
$lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryLibCSrcPath);
# Rebuild if the modification time of the smart file does equal the last changes in source paths
if ($bSmart)
{
if (!$oStorageBackRest->exists($strLibCSmart) || $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
{
&log(INFO, 'libc dependencies have changed, rebuilding...');
&log(INFO, ' libc dependencies have changed, rebuilding...');
$bRebuild = true;
}
@ -600,7 +595,7 @@ eval
if ($bRebuild)
{
&log(INFO, "build C library for ${strBuildVM} (${strBuildPath})");
&log(INFO, " build C library for ${strBuildVM} (${strBuildPath})");
# It's very expensive to rebuild the Makefile so make sure it has actually changed
my $bMakeRebuild =