Improve efficiency of C library builds now that they are used only for testing.

doc/xml/release.xml

@@ -27,6 +27,10 @@
 
             <release-test-list>
                 <release-development-list>
+                    <release-item>
+                        <p>Improve efficiency of C library builds now that they are used only for testing.</p>
+                    </release-item>
+
                     <release-item>
                         <p>Remove RHEL package patch since it has been committed upstream.</p>
                     </release-item>

libc/Makefile.PL

@@ -15,14 +15,11 @@ use ExtUtils::MakeMaker;
 use File::Basename qw(dirname);
 
 use lib dirname($0) . '/lib';
-use lib dirname($0) . '/build/lib';
 
 ####################################################################################################################################
 # Storage object to use for all file operations
 ####################################################################################################################################
 use constant BACKREST_NAME                                          => 'pgBackRest';
-use constant LIB_NAME                                               => 'LibC';
-use constant LIB_AUTO_NAME                                          => LIB_NAME . 'Auto';
 
 ####################################################################################################################################
 # Create C Makefile
@@ -75,28 +72,10 @@ my @stryCFile =
     'storage/storage.c',
 );
 
-# Link source files to build the C library.  The library has different build options and we don't want the bin build to reuse any of
+# Add ../src for files that are outside libc
-# them.  Also, this makes the output of the __FILE__ macro prettier.
 for (my $iFileIdx = 1; $iFileIdx < @stryCFile; $iFileIdx++)
 {
-    # Make directory
+    $stryCFile[$iFileIdx] = '../src/' . $stryCFile[$iFileIdx];
-    system("mkdir -p " . dirname($stryCFile[$iFileIdx])) == 0
-        or die('unable to mkdir ' . dirname($stryCFile[$iFileIdx]));
-
-    # Link the file to the source directory
-    my @stryLink = split('/', $stryCFile[$iFileIdx]);
-    my $strLink;
-
-    for (my $iLinkIdx = 0; $iLinkIdx < @stryLink; $iLinkIdx++)
-    {
-        $strLink .= (defined($strLink) ? '/' : '') . '..';
-    }
-
-    if (!-l $stryCFile[$iFileIdx])
-    {
-        system("ln -s ${strLink}/src/$stryCFile[$iFileIdx] $stryCFile[$iFileIdx]") == 0
-            or die("unable to link ../src/$stryCFile[$iFileIdx] to $stryCFile[$iFileIdx]");
-    }
 }
 
 # Write the makefile

libc/t/sanity.t

@@ -1,35 +0,0 @@
-####################################################################################################################################
-# Sanity Tests for C Library
-#
-# Test to ensure the C library loads and is compiled correctly.  Unit and integration tests are performed by test/test.pl.
-####################################################################################################################################
-use strict;
-use warnings;
-use Carp;
-use English '-no_match_vars';
-
-use Cwd qw(abs_path);
-use File::Basename qw(dirname);
-
-# Set number of tests
-use Test::More tests => 5;
-
-use lib abs_path(dirname($0) . '/../../lib');
-
-# Make sure the module loads without errors
-BEGIN {use_ok('pgBackRest::LibC', qw(:debug :config :configDefine))};
-
-require XSLoader;
-XSLoader::load('pgBackRest::LibC', '999');
-
-# UVSIZE determines the pointer and long long int size.  This needs to be 8 to indicate 64-bit types are available.
-ok (libcUvSize() == 8, 'UVSIZE == 8');
-
-# Check constant that is created dynamically
-ok (CFGOPTVAL_BACKUP_TYPE_FULL eq 'full', 'auto constant valid');
-
-# Check constant that is exported from C
-ok (CFGDEF_TYPE_HASH >= 0, 'auto constant valid');
-
-# Check a function
-ok (cfgOptionName(CFGOPT_DELTA) eq 'delta', 'auto constant valid');

test/lib/pgBackRestTest/Common/BuildTest.pm

@@ -222,4 +222,27 @@ sub buildMakefile
 
 push @EXPORT, qw(buildMakefile);
 
+####################################################################################################################################
+# Update a Makefile with object compile rules
+####################################################################################################################################
+sub buildLoadLibC
+{
+    # Load the module dynamically
+    require pgBackRest::LibC;
+    pgBackRest::LibC->import(qw(:debug));
+
+    # Load shared object
+    require XSLoader;
+    XSLoader::load('pgBackRest::LibC', '999');
+
+    # Do a basic test to make sure it installed correctly
+    if (libcUvSize() != 8)
+    {
+        confess &log(ERROR, 'UVSIZE in test library does not equal 8');
+    }
+}
+
+push @EXPORT, qw(buildLoadLibC);
+
+
 1;

test/lib/pgBackRestTest/Common/JobTest.pm

@@ -625,9 +625,9 @@ sub jobInstallC
 
     # Install Perl C Library
     my $oVm = vmGet();
-    my $strBuildPath = "${strBasePath}/test/.vagrant";
+    my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}";
-    my $strBuildLibCPath = "$strBuildPath/libc/${strVm}/libc";
+    my $strBuildLibCPath = "${strBuildPath}/libc";
-    my $strBuildBinPath = "$strBuildPath/bin/${strVm}/src";
+    my $strBuildBinPath = "${strBuildPath}/src";
     my $strPerlAutoPath = $oVm->{$strVm}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC';
 
     executeTest(

test/test.pl

@@ -663,146 +663,14 @@ eval
             my $oVm = vmGet();
             my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
             my $lTimestampLast;
-
+            my @stryBinSrcPath = ('src', 'libc');
-            # Build the C Library
+            my $strBinPath = "${strVagrantPath}/bin";
-            #-----------------------------------------------------------------------------------------------------------------------
+            my $rhBinBuild = {};
-            if ($bLibCHostRequired || $bLibCVmRequired)
-            {
-                my $strLibCPath = "${strVagrantPath}/libc";
-                my $strLibCSmart = "${strLibCPath}/build.timestamp";
-                my $bRebuild = !$bSmart;
-                my @stryLibCSrcPath = $bLibCHostRequired || $bLibCVmRequired ? ('libc', 'src') : ('libc');
-
-                # Find the lastest modified time for dirs that affect the libc build
-                $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryLibCSrcPath);
-
-                # Rebuild if the modification time of the smart file does equal the last changes in source paths
-                if ($bSmart)
-                {
-                    if (!$oStorageBackRest->exists($strLibCSmart) ||
-                        $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
-                    {
-                        &log(INFO, '    libc dependencies have changed, rebuilding...');
-
-                        $bRebuild = true;
-                    }
-                }
-                else
-                {
-                    executeTest("sudo rm -rf ${strLibCPath}");
-                }
-
-                # Delete old libc files from the host
-                if ($bRebuild)
-                {
-                    executeTest('sudo rm -rf ' . $oVm->{$strVmHost}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC');
-                    executeTest('sudo rm -rf ' . $oVm->{$strVmHost}{&VMDEF_PERL_ARCH_PATH} . '/pgBackRest');
-                }
-
-                # Loop through VMs to do the C Library builds
-                my $bLogDetail = $strLogLevel eq 'detail';
-                my @stryBuildVm = ();
-
-                if ($strVm eq VM_ALL)
-                {
-                    @stryBuildVm  = $bLibCVmRequired ? VM_LIST : ($strVmHost);
-                }
-                else
-                {
-                    @stryBuildVm  = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost);
-                }
-
-                foreach my $strBuildVM (@stryBuildVm)
-                {
-                    my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc";
-                    my $bContainerExists = $strBuildVM ne $strVmHost;
-
-                    if ($bRebuild)
-                    {
-                        &log(INFO, "    build test library for ${strBuildVM} (${strBuildPath})");
-
-                        # It's very expensive to rebuild the Makefile so make sure it has actually changed
-                        my $bMakeRebuild =
-                            !$oStorageBackRest->exists("${strBuildPath}/Makefile") ||
-                            ($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime >
-                             $oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime);
-
-                        if ($bContainerExists)
-                        {
-                            executeTest(
-                                "docker run -itd -h test-build --name=test-build" .
-                                " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build",
-                                {bSuppressStdErr => true});
-                        }
-
-                        foreach my $strLibCSrcPath ('lib', 'libc', 'src')
-                        {
-                            $oStorageBackRest->pathCreate(
-                                "${strLibCPath}/${strBuildVM}/${strLibCSrcPath}", {bIgnoreExists => true, bCreateParent => true});
-                            executeTest(
-                                "rsync -rt ${strBackRestBase}/${strLibCSrcPath}/* ${strLibCPath}/${strBuildVM}/${strLibCSrcPath}");
-                        }
-
-                        if ($bMakeRebuild)
-                        {
-                            executeTest(
-                                ($bContainerExists ? "docker exec -i test-build bash -c '" : '') .
-                                "cd ${strBuildPath} && perl Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" .
-                                ($bContainerExists ? "'" : ''),
-                                {bSuppressStdErr => true, bShowOutputAsync => $bLogDetail});
-                        }
-
-                        executeTest(
-                            ($bContainerExists ? 'docker exec -i test-build ' : '') .
-                                "make --silent --directory ${strBuildPath}",
-                            {bShowOutputAsync => $bLogDetail});
-                        executeTest(
-                            ($bContainerExists ? 'docker exec -i test-build ' : '') .
-                                "make --silent --directory ${strBuildPath} test",
-                            {bShowOutputAsync => $bLogDetail});
-                        executeTest(
-                            ($bContainerExists ? 'docker exec -i test-build ' : 'sudo ') .
-                                "make --silent --directory ${strBuildPath} install",
-                            {bShowOutputAsync => $bLogDetail});
-
-                        if ($bContainerExists)
-                        {
-                            executeTest("docker rm -f test-build");
-                        }
-
-                        if ($strBuildVM eq $strVmHost)
-                        {
-                            executeTest("sudo make -C ${strBuildPath} install", {bSuppressStdErr => true});
-
-                            # Load the module dynamically
-                            require pgBackRest::LibC;
-                            pgBackRest::LibC->import(qw(:debug));
-
-                            require XSLoader;
-                            XSLoader::load('pgBackRest::LibC', '999');
-
-                            # Do a basic test to make sure it installed correctly
-                            if (libcUvSize() != 8)
-                            {
-                                confess &log(ERROR, 'UVSIZE in test library does not equal 8');
-                            }
-                        }
-                    }
-                }
-
-                # Write files to indicate the last time a build was successful
-                $oStorageBackRest->put($strLibCSmart);
-                utime($lTimestampLast, $lTimestampLast, $strLibCSmart) or
-                    confess "unable to set time for ${strLibCSmart}" . (defined($!) ? ":$!" : '');
-            }
 
             # Build the binary
             #-----------------------------------------------------------------------------------------------------------------------
             if ($bBinRequired)
             {
-                my $strBinPath = "${strVagrantPath}/bin";
-                my @stryBinSrcPath = ('src', 'libc');
-
                 # Find the lastest modified time for dirs that affect the bin build
                 $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryBinSrcPath);
 
@@ -814,6 +682,7 @@ eval
                 {
                     my $strBuildPath = "${strBinPath}/${strBuildVM}/src";
                     my $bRebuild = !$bSmart;
+                    $rhBinBuild->{$strBuildVM} = true;
 
                     # Rebuild if the modification time of the smart file does equal the last changes in source paths
                     if ($bSmart)
@@ -855,7 +724,8 @@ eval
                         }
 
                         my $strCExtra =
-                            "'-g" . (vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace ? ' -DWITH_BACKTRACE' : '') . "'";
+                            "'-g -fPIC -D_FILE_OFFSET_BITS=64" .
+                            (vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace ? ' -DWITH_BACKTRACE' : '') . "'";
                         my $strLdExtra = vmWithBackTrace($strBuildVM) && $bNoLint && $bBackTrace  ? '-lbacktrace' : '';
                         my $strCDebug = vmDebugIntegration($strBuildVM) ? 'CDEBUG=' : '';
 
@@ -870,6 +740,117 @@ eval
                 }
             }
 
+            # Build the C Library
+            #-----------------------------------------------------------------------------------------------------------------------
+            if ($bLibCHostRequired || $bLibCVmRequired)
+            {
+                my $strLibCPath = "${strVagrantPath}/bin";
+
+                # Loop through VMs to do the C Library builds
+                my $bLogDetail = $strLogLevel eq 'detail';
+                my @stryBuildVm = ();
+
+                if ($strVm eq VM_ALL)
+                {
+                    @stryBuildVm  = $bLibCVmRequired ? VM_LIST : ($strVmHost);
+                }
+                else
+                {
+                    @stryBuildVm  = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost);
+                }
+
+                foreach my $strBuildVM (@stryBuildVm)
+                {
+                    my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc";
+                    my $bContainerExists = $strBuildVM ne $strVmHost;
+
+                    my $strLibCSmart = "${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so";
+                    my $bRebuild = !$bSmart;
+
+                    # Rebuild if the modification time of the smart file does equal the last changes in source paths
+                    if ($bSmart)
+                    {
+                        if (!$oStorageBackRest->exists($strLibCSmart) ||
+                            $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
+                        {
+                            &log(INFO, '    libc dependencies have changed for ${strBuildVM}, rebuilding...');
+
+                            $bRebuild = true;
+                        }
+                    }
+
+                    # Delete old libc files from the host
+                    if ($bRebuild)
+                    {
+                        executeTest('sudo rm -rf ' . $oVm->{$strBuildVM}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC');
+                    }
+
+                    if ($bRebuild)
+                    {
+                        &log(INFO, "    build test library for ${strBuildVM} (${strBuildPath})");
+
+                        if (!$rhBinBuild->{$strBuildVM})
+                        {
+                            foreach my $strBinSrcPath (@stryBinSrcPath)
+                            {
+                                $oStorageBackRest->pathCreate(
+                                    "${strBinPath}/${strBuildVM}/${strBinSrcPath}", {bIgnoreExists => true, bCreateParent => true});
+                            }
+
+                            executeTest(
+                                "rsync -rt" . (!$bSmart ? " --delete-excluded" : '') .
+                                " --include=" . join('/*** --include=', @stryBinSrcPath) . '/*** --exclude=*' .
+                                " ${strBackRestBase}/ ${strBinPath}/${strBuildVM}");
+                        }
+
+                        # Can't reuse any object files in the libc dir because it does not have proper dependencies
+                        executeTest(
+                            "rsync -rt --exclude=Makefile --delete ${strBackRestBase}/libc/ ${strLibCPath}/${strBuildVM}/libc");
+
+                        # It's very expensive to rebuild the Makefile so make sure it has actually changed
+                        my $bMakeRebuild =
+                            !$oStorageBackRest->exists("${strBuildPath}/Makefile") ||
+                            ($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime >
+                             $oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime);
+
+                        if ($bContainerExists)
+                        {
+                            executeTest(
+                                "docker run -itd -h test-build --name=test-build" .
+                                " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build",
+                                {bSuppressStdErr => true});
+                        }
+
+                        if ($bMakeRebuild)
+                        {
+                            &log(INFO, "    rebuild test library Makefile for ${strBuildVM}");
+
+                            executeTest(
+                                ($bContainerExists ? "docker exec -i test-build bash -c '" : '') .
+                                "cd ${strBuildPath} && perl Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" .
+                                ($bContainerExists ? "'" : ''),
+                                {bSuppressStdErr => true, bShowOutputAsync => $bLogDetail});
+                        }
+
+                        executeTest(
+                            ($bContainerExists ? 'docker exec -i test-build ' : '') .
+                                "make --silent --directory ${strBuildPath}",
+                            {bShowOutputAsync => $bLogDetail});
+
+                        if ($bContainerExists)
+                        {
+                            executeTest("docker rm -f test-build");
+                        }
+
+                        if ($strBuildVM eq $strVmHost)
+                        {
+                            executeTest("sudo make -C ${strBuildPath} install", {bSuppressStdErr => true});
+                            buildLoadLibC();
+                        }
+                    }
+                }
+            }
+
             # Build the package
             #-----------------------------------------------------------------------------------------------------------------------
             if (!$bNoPackage)
@@ -1409,17 +1390,11 @@ eval
         exit 0;
     }
 
-    # Load the module dynamically
-    require pgBackRest::LibCAuto;
-    require pgBackRest::LibC;
-    pgBackRest::LibC->import(qw(:debug));
-
-    require XSLoader;
-    XSLoader::load('pgBackRest::LibC', '999');
-
     ################################################################################################################################
     # Runs tests
     ################################################################################################################################
+    buildLoadLibC();
+
     my $iRun = 0;
 
     # Create host group for containers