diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 7a8e68c3d..eb826672e 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -27,6 +27,10 @@ + +

Improve efficiency of C library builds now that they are used only for testing.

+ +

Remove RHEL package patch since it has been committed upstream.

 diff --git a/libc/Makefile.PL b/libc/Makefile.PL index 3c6a36219..dfc09b90c 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -15,14 +15,11 @@ use ExtUtils::MakeMaker; use File::Basename qw(dirname); use lib dirname($0) . '/lib'; -use lib dirname($0) . '/build/lib'; #################################################################################################################################### # Storage object to use for all file operations #################################################################################################################################### use constant BACKREST_NAME => 'pgBackRest'; -use constant LIB_NAME => 'LibC'; -use constant LIB_AUTO_NAME => LIB_NAME . 'Auto'; #################################################################################################################################### # Create C Makefile @@ -75,28 +72,10 @@ my @stryCFile = 'storage/storage.c', ); -# Link source files to build the C library. The library has different build options and we don't want the bin build to reuse any of -# them. Also, this makes the output of the __FILE__ macro prettier. +# Add ../src for files that are outside libc for (my $iFileIdx = 1; $iFileIdx < @stryCFile; $iFileIdx++) { - # Make directory - system("mkdir -p " . dirname($stryCFile[$iFileIdx])) == 0 - or die('unable to mkdir ' . dirname($stryCFile[$iFileIdx])); - - # Link the file to the source directory - my @stryLink = split('/', $stryCFile[$iFileIdx]); - my $strLink; - - for (my $iLinkIdx = 0; $iLinkIdx < @stryLink; $iLinkIdx++) - { - $strLink .= (defined($strLink) ? '/' : '') . '..'; - } - - if (!-l $stryCFile[$iFileIdx]) - { - system("ln -s ${strLink}/src/$stryCFile[$iFileIdx] $stryCFile[$iFileIdx]") == 0 - or die("unable to link ../src/$stryCFile[$iFileIdx] to $stryCFile[$iFileIdx]"); - } + $stryCFile[$iFileIdx] = '../src/' . $stryCFile[$iFileIdx]; } # Write the makefile diff --git a/libc/t/sanity.t b/libc/t/sanity.t deleted file mode 100644 index 4d37ef937..000000000 --- a/libc/t/sanity.t +++ /dev/null @@ -1,35 +0,0 @@ -#################################################################################################################################### -# Sanity Tests for C Library -# -# Test to ensure the C library loads and is compiled correctly. Unit and integration tests are performed by test/test.pl. -#################################################################################################################################### -use strict; -use warnings; -use Carp; -use English '-no_match_vars'; - -use Cwd qw(abs_path); -use File::Basename qw(dirname); - -# Set number of tests -use Test::More tests => 5; - -use lib abs_path(dirname($0) . '/../../lib'); - -# Make sure the module loads without errors -BEGIN {use_ok('pgBackRest::LibC', qw(:debug :config :configDefine))}; - -require XSLoader; -XSLoader::load('pgBackRest::LibC', '999'); - -# UVSIZE determines the pointer and long long int size. This needs to be 8 to indicate 64-bit types are available. -ok (libcUvSize() == 8, 'UVSIZE == 8'); - -# Check constant that is created dynamically -ok (CFGOPTVAL_BACKUP_TYPE_FULL eq 'full', 'auto constant valid'); - -# Check constant that is exported from C -ok (CFGDEF_TYPE_HASH >= 0, 'auto constant valid'); - -# Check a function -ok (cfgOptionName(CFGOPT_DELTA) eq 'delta', 'auto constant valid'); diff --git a/test/lib/pgBackRestTest/Common/BuildTest.pm b/test/lib/pgBackRestTest/Common/BuildTest.pm index 55c428b05..51641c4ad 100644 --- a/test/lib/pgBackRestTest/Common/BuildTest.pm +++ b/test/lib/pgBackRestTest/Common/BuildTest.pm @@ -222,4 +222,27 @@ sub buildMakefile push @EXPORT, qw(buildMakefile); +#################################################################################################################################### +# Update a Makefile with object compile rules +#################################################################################################################################### +sub buildLoadLibC +{ + # Load the module dynamically + require pgBackRest::LibC; + pgBackRest::LibC->import(qw(:debug)); + + # Load shared object + require XSLoader; + XSLoader::load('pgBackRest::LibC', '999'); + + # Do a basic test to make sure it installed correctly + if (libcUvSize() != 8) + { + confess &log(ERROR, 'UVSIZE in test library does not equal 8'); + } +} + +push @EXPORT, qw(buildLoadLibC); + + 1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 0bf7514dc..30a12ad81 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -625,9 +625,9 @@ sub jobInstallC # Install Perl C Library my $oVm = vmGet(); - my $strBuildPath = "${strBasePath}/test/.vagrant"; - my $strBuildLibCPath = "$strBuildPath/libc/${strVm}/libc"; - my $strBuildBinPath = "$strBuildPath/bin/${strVm}/src"; + my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}"; + my $strBuildLibCPath = "${strBuildPath}/libc"; + my $strBuildBinPath = "${strBuildPath}/src"; my $strPerlAutoPath = $oVm->{$strVm}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC'; executeTest( diff --git a/test/test.pl b/test/test.pl index ce4a25bad..7ab56c4d6 100755 --- a/test/test.pl +++ b/test/test.pl @@ -663,146 +663,14 @@ eval my $oVm = vmGet(); my $strVagrantPath = "${strBackRestBase}/test/.vagrant"; my $lTimestampLast; - - # Build the C Library - #----------------------------------------------------------------------------------------------------------------------- - if ($bLibCHostRequired || $bLibCVmRequired) - { - my $strLibCPath = "${strVagrantPath}/libc"; - my $strLibCSmart = "${strLibCPath}/build.timestamp"; - my $bRebuild = !$bSmart; - my @stryLibCSrcPath = $bLibCHostRequired || $bLibCVmRequired ? ('libc', 'src') : ('libc'); - - # Find the lastest modified time for dirs that affect the libc build - $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryLibCSrcPath); - - # Rebuild if the modification time of the smart file does equal the last changes in source paths - if ($bSmart) - { - if (!$oStorageBackRest->exists($strLibCSmart) || - $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast) - { - &log(INFO, ' libc dependencies have changed, rebuilding...'); - - $bRebuild = true; - } - } - else - { - executeTest("sudo rm -rf ${strLibCPath}"); - } - - # Delete old libc files from the host - if ($bRebuild) - { - executeTest('sudo rm -rf ' . $oVm->{$strVmHost}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC'); - executeTest('sudo rm -rf ' . $oVm->{$strVmHost}{&VMDEF_PERL_ARCH_PATH} . '/pgBackRest'); - } - - # Loop through VMs to do the C Library builds - my $bLogDetail = $strLogLevel eq 'detail'; - my @stryBuildVm = (); - - if ($strVm eq VM_ALL) - { - @stryBuildVm = $bLibCVmRequired ? VM_LIST : ($strVmHost); - } - else - { - @stryBuildVm = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost); - } - - foreach my $strBuildVM (@stryBuildVm) - { - my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc"; - my $bContainerExists = $strBuildVM ne $strVmHost; - - if ($bRebuild) - { - &log(INFO, " build test library for ${strBuildVM} (${strBuildPath})"); - - # It's very expensive to rebuild the Makefile so make sure it has actually changed - my $bMakeRebuild = - !$oStorageBackRest->exists("${strBuildPath}/Makefile") || - ($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime > - $oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime); - - if ($bContainerExists) - { - executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build", - {bSuppressStdErr => true}); - } - - foreach my $strLibCSrcPath ('lib', 'libc', 'src') - { - $oStorageBackRest->pathCreate( - "${strLibCPath}/${strBuildVM}/${strLibCSrcPath}", {bIgnoreExists => true, bCreateParent => true}); - executeTest( - "rsync -rt ${strBackRestBase}/${strLibCSrcPath}/* ${strLibCPath}/${strBuildVM}/${strLibCSrcPath}"); - } - - if ($bMakeRebuild) - { - executeTest( - ($bContainerExists ? "docker exec -i test-build bash -c '" : '') . - "cd ${strBuildPath} && perl Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" . - ($bContainerExists ? "'" : ''), - {bSuppressStdErr => true, bShowOutputAsync => $bLogDetail}); - } - - executeTest( - ($bContainerExists ? 'docker exec -i test-build ' : '') . - "make --silent --directory ${strBuildPath}", - {bShowOutputAsync => $bLogDetail}); - executeTest( - ($bContainerExists ? 'docker exec -i test-build ' : '') . - "make --silent --directory ${strBuildPath} test", - {bShowOutputAsync => $bLogDetail}); - executeTest( - ($bContainerExists ? 'docker exec -i test-build ' : 'sudo ') . - "make --silent --directory ${strBuildPath} install", - {bShowOutputAsync => $bLogDetail}); - - if ($bContainerExists) - { - executeTest("docker rm -f test-build"); - } - - if ($strBuildVM eq $strVmHost) - { - executeTest("sudo make -C ${strBuildPath} install", {bSuppressStdErr => true}); - - # Load the module dynamically - require pgBackRest::LibC; - pgBackRest::LibC->import(qw(:debug)); - - require XSLoader; - XSLoader::load('pgBackRest::LibC', '999'); - - # Do a basic test to make sure it installed correctly - if (libcUvSize() != 8) - { - confess &log(ERROR, 'UVSIZE in test library does not equal 8'); - } - } - } - } - - # Write files to indicate the last time a build was successful - $oStorageBackRest->put($strLibCSmart); - utime($lTimestampLast, $lTimestampLast, $strLibCSmart) or - confess "unable to set time for ${strLibCSmart}" . (defined($!) ? ":$!" : ''); - } + my @stryBinSrcPath = ('src', 'libc'); + my $strBinPath = "${strVagrantPath}/bin"; + my $rhBinBuild = {}; # Build the binary #----------------------------------------------------------------------------------------------------------------------- if ($bBinRequired) { - my $strBinPath = "${strVagrantPath}/bin"; - my @stryBinSrcPath = ('src', 'libc'); - # Find the lastest modified time for dirs that affect the bin build $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryBinSrcPath); @@ -814,6 +682,7 @@ eval { my $strBuildPath = "${strBinPath}/${strBuildVM}/src"; my $bRebuild = !$bSmart; + $rhBinBuild->{$strBuildVM} = true; # Rebuild if the modification time of the smart file does equal the last changes in source paths if ($bSmart) @@ -855,7 +724,8 @@ eval } my $strCExtra = - "'-g" . (vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace ? ' -DWITH_BACKTRACE' : '') . "'"; + "'-g -fPIC -D_FILE_OFFSET_BITS=64" . + (vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace ? ' -DWITH_BACKTRACE' : '') . "'"; my $strLdExtra = vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace ? '-lbacktrace' : ''; my $strCDebug = vmDebugIntegration($strBuildVM) ? 'CDEBUG=' : ''; @@ -870,6 +740,117 @@ eval } } + # Build the C Library + #----------------------------------------------------------------------------------------------------------------------- + if ($bLibCHostRequired || $bLibCVmRequired) + { + my $strLibCPath = "${strVagrantPath}/bin"; + + # Loop through VMs to do the C Library builds + my $bLogDetail = $strLogLevel eq 'detail'; + my @stryBuildVm = (); + + if ($strVm eq VM_ALL) + { + @stryBuildVm = $bLibCVmRequired ? VM_LIST : ($strVmHost); + } + else + { + @stryBuildVm = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost); + } + + foreach my $strBuildVM (@stryBuildVm) + { + my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc"; + my $bContainerExists = $strBuildVM ne $strVmHost; + + my $strLibCSmart = "${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so"; + my $bRebuild = !$bSmart; + + # Rebuild if the modification time of the smart file does equal the last changes in source paths + if ($bSmart) + { + if (!$oStorageBackRest->exists($strLibCSmart) || + $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast) + { + &log(INFO, ' libc dependencies have changed for ${strBuildVM}, rebuilding...'); + + $bRebuild = true; + } + } + + # Delete old libc files from the host + if ($bRebuild) + { + executeTest('sudo rm -rf ' . $oVm->{$strBuildVM}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC'); + } + + if ($bRebuild) + { + &log(INFO, " build test library for ${strBuildVM} (${strBuildPath})"); + + if (!$rhBinBuild->{$strBuildVM}) + { + foreach my $strBinSrcPath (@stryBinSrcPath) + { + $oStorageBackRest->pathCreate( + "${strBinPath}/${strBuildVM}/${strBinSrcPath}", {bIgnoreExists => true, bCreateParent => true}); + } + + executeTest( + "rsync -rt" . (!$bSmart ? " --delete-excluded" : '') . + " --include=" . join('/*** --include=', @stryBinSrcPath) . '/*** --exclude=*' . + " ${strBackRestBase}/ ${strBinPath}/${strBuildVM}"); + } + + # Can't reuse any object files in the libc dir because it does not have proper dependencies + executeTest( + "rsync -rt --exclude=Makefile --delete ${strBackRestBase}/libc/ ${strLibCPath}/${strBuildVM}/libc"); + + # It's very expensive to rebuild the Makefile so make sure it has actually changed + my $bMakeRebuild = + !$oStorageBackRest->exists("${strBuildPath}/Makefile") || + ($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime > + $oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime); + + if ($bContainerExists) + { + executeTest( + "docker run -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build", + {bSuppressStdErr => true}); + } + + if ($bMakeRebuild) + { + &log(INFO, " rebuild test library Makefile for ${strBuildVM}"); + + executeTest( + ($bContainerExists ? "docker exec -i test-build bash -c '" : '') . + "cd ${strBuildPath} && perl Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" . + ($bContainerExists ? "'" : ''), + {bSuppressStdErr => true, bShowOutputAsync => $bLogDetail}); + } + + executeTest( + ($bContainerExists ? 'docker exec -i test-build ' : '') . + "make --silent --directory ${strBuildPath}", + {bShowOutputAsync => $bLogDetail}); + + if ($bContainerExists) + { + executeTest("docker rm -f test-build"); + } + + if ($strBuildVM eq $strVmHost) + { + executeTest("sudo make -C ${strBuildPath} install", {bSuppressStdErr => true}); + buildLoadLibC(); + } + } + } + } + # Build the package #----------------------------------------------------------------------------------------------------------------------- if (!$bNoPackage) @@ -1409,17 +1390,11 @@ eval exit 0; } - # Load the module dynamically - require pgBackRest::LibCAuto; - require pgBackRest::LibC; - pgBackRest::LibC->import(qw(:debug)); - - require XSLoader; - XSLoader::load('pgBackRest::LibC', '999'); - ################################################################################################################################ # Runs tests ################################################################################################################################ + buildLoadLibC(); + my $iRun = 0; # Create host group for containers