Improve code generation performance.

build/lib/pgBackRestBuild/Build.pm

@@ -162,25 +162,8 @@ sub buildAll
                     $strExt = $strFileType eq BLD_C ? $strFileExt : "${strFileExt}h";
                 }
 
-                # Only save the file if the content has changed
+                # Save the file
-                my $strFileName = "${strPath}/${strFile}.auto.${strExt}";
+                $oStorage->put("${strPath}/${strFile}.auto.${strExt}", trim($rhSource->{$strFileType}) . "\n");
-                my $strContent = trim($rhSource->{$strFileType}) . "\n";
-                my $bChanged = true;
-
-                if ($oStorage->exists($strFileName))
-                {
-                    my $strOldContent = ${$oStorage->get($strFileName)};
-
-                    if ($strContent eq $strOldContent)
-                    {
-                        $bChanged = false;
-                    }
-                }
-
-                if ($bChanged)
-                {
-                    $oStorage->put($strFileName, $strContent);
-                }
             }
         }
     }

doc/xml/release.xml

@@ -110,7 +110,7 @@
 
                 <release-development-list>
                     <release-item>
-                        <p>Improve bin and libc build performance.</p>
+                        <p>Buld performance improvements.  Improve bin and libc build performance.  Improve code generation performance.</p>
                     </release-item>
 
                     <release-item>

libc/build/lib/pgBackRestLibC/Build.pm

@@ -297,24 +297,8 @@ sub buildXsAll
         "\n" .
         "1;\n";
 
-    # Only save the file if it has changed
+    # Save the file
-    my $strLibFile = 'lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm';
+    $oStorage->put('lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm', $strContent);
-    my $bChanged = true;
-
-    if ($oStorage->exists($strLibFile))
-    {
-        my $strContentOld = ${$oStorage->get($strLibFile)};
-
-        if ($strContent eq $strContentOld)
-        {
-            $bChanged = false;
-        }
-    }
-
-    if ($bChanged)
-    {
-        $oStorage->put($strLibFile, $strContent);
-    }
 
     # Build error file
     #-------------------------------------------------------------------------------------------------------------------------------

test/lib/pgBackRestTest/Common/BuildTest.pm

@@ -0,0 +1,57 @@
+####################################################################################################################################
+# Build Binaries and Auto-Generate Code
+####################################################################################################################################
+package pgBackRestTest::Common::BuildTest;
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+# use Cwd qw(abs_path);
+use Exporter qw(import);
+    our @EXPORT = qw();
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Find last modification time in a list of directories, with optional filters
+####################################################################################################################################
+sub buildLastModTime
+{
+    my $oStorage = shift;
+    my $strBasePath = shift;
+    my $rstrySubPath = shift;
+    my $strPattern = shift;
+
+    my $lTimestampLast = 0;
+
+    foreach my $strSubPath (defined($rstrySubPath) ? @{$rstrySubPath} : (''))
+    {
+        my $hManifest = $oStorage->manifest($strBasePath . ($strSubPath eq '' ? '' : "/${strSubPath}"));
+
+        foreach my $strFile (sort(keys(%{$hManifest})))
+        {
+            next if (defined($strPattern) && $strFile !~ /$strPattern/);
+
+            if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
+            {
+                $lTimestampLast = $hManifest->{$strFile}{modification_time};
+            }
+        }
+    }
+
+    if ($lTimestampLast == 0)
+    {
+        confess &log(ERROR, "no files found");
+    }
+
+    return $lTimestampLast;
+}
+
+push @EXPORT, qw(buildLastModTime);
+
+1;

test/test.pl

@@ -47,6 +47,7 @@ use pgBackRestBuild::Error::Data;
 
 use BackRestDoc::Custom::DocCustomRelease;
 
+use pgBackRestTest::Common::BuildTest;
 use pgBackRestTest::Common::ContainerTest;
 use pgBackRestTest::Common::CiTest;
 use pgBackRestTest::Common::DefineTest;
@@ -238,6 +239,7 @@ eval
     }
 
     logLevelSet(uc($strLogLevel), uc($strLogLevel), OFF);
+    &log(INFO, "test begin - log level ${strLogLevel}");
 
     if (@stryModuleTest != 0 && @stryModule != 1)
     {
@@ -303,9 +305,15 @@ eval
     {
         # Auto-generate C files
         #---------------------------------------------------------------------------------------------------------------------------
-        &log(INFO, "autogenerate C code");
+        &log(INFO, "check code autogenerate");
+
         errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")});
 
+        if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['build']) >
+            buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['src'], '\.auto\.c$'))
+        {
+            &log(INFO, "    autogenerate C code");
+
             my $rhBuild =
             {
                 'config' =>
@@ -334,16 +342,21 @@ eval
             };
 
             buildAll("${strBackRestBase}/src", $rhBuild);
+        }
 
         # Auto-generate XS files
         #
         # Use statements are put here so this will be easy to get rid of someday.
         #---------------------------------------------------------------------------------------------------------------------------
-        &log(INFO, "autogenerate Perl code");
+        if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc/build']) >
+            buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc', 'lib'], '(\.auto\.pm|Auto\.pm)$'))
+        {
+            &log(INFO, "    autogenerate Perl code");
             use lib dirname(dirname($0)) . '/libc/build/lib';
             use pgBackRestLibC::Build;                                      ## no critic (Modules::ProhibitConditionalUseStatements)
 
             buildXsAll("${strBackRestBase}/libc");
+        }
 
         if ($bGenOnly)
         {
@@ -351,6 +364,7 @@ eval
         }
 
         # Sync time to prevent build failures when running on VirtualBox.
+        &log(INFO, "sync vbox time");
         my $strVBoxService = '/usr/sbin/VBoxService';
 
         if ($oStorageTest->exists($strVBoxService))
@@ -365,6 +379,8 @@ eval
             (new pgBackRestTest::Common::CiTest($oStorageBackRest))->process();
         }
 
+        &log(INFO, "check version info");
+
         # Load the doc module dynamically since it is not supported on all systems
         require BackRestDoc::Common::Doc;
         BackRestDoc::Common::Doc->import();
@@ -421,6 +437,7 @@ eval
 
         if (!$bDryRun || $bVmOut)
         {
+            &log(INFO, "cleanup old data and containers");
             containerRemove('test-([0-9]+|build)');
 
             for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++)
@@ -464,6 +481,8 @@ eval
         #---------------------------------------------------------------------------------------------------------------------------
         if (!$bDryRun && $bBuildRequired)
         {
+            &log(INFO, "check bin builds");
+
             my $oVm = vmGet();
             my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
 
@@ -475,27 +494,14 @@ eval
             my @stryBinSrcPath = ('src');
 
             # Find the lastest modified time for dirs that affect the bin build
-            my $lTimestampLast = $oStorageBackRest->exists($strBinSmart) ? $oStorageBackRest->info($strBinSmart)->mtime : 0;
+            my $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryBinSrcPath);
-
-            foreach my $strBinSrcPath (@stryBinSrcPath)
-            {
-                my $hManifest = $oStorageBackRest->manifest($strBinSrcPath);
-
-                foreach my $strFile (sort(keys(%{$hManifest})))
-                {
-                    if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
-                    {
-                        $lTimestampLast = $hManifest->{$strFile}{modification_time};
-                    }
-                }
-            }
 
             # Rebuild if the modification time of the smart file does equal the last changes in source paths
             if ($bSmart)
             {
                 if (!$oStorageBackRest->exists($strBinSmart) || $oStorageBackRest->info($strBinSmart)->mtime < $lTimestampLast)
                 {
-                    &log(INFO, 'bin dependencies have changed, rebuilding...');
+                    &log(INFO, '    bin dependencies have changed, rebuilding...');
 
                     $bRebuild = true;
                 }
@@ -515,7 +521,7 @@ eval
 
                 if ($bRebuild)
                 {
-                    &log(INFO, "build bin for ${strBuildVM} (${strBuildPath})");
+                    &log(INFO, "    build bin for ${strBuildVM} (${strBuildPath})");
 
                     executeTest(
                         "docker run -itd -h test-build --name=test-build" .
@@ -554,25 +560,14 @@ eval
             my @stryLibCSrcPath = ('libc', 'src');
 
             # Find the lastest modified time for dirs that affect the libc build
-            foreach my $strLibCSrcPath (@stryLibCSrcPath)
+            $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryLibCSrcPath);
-            {
-                my $hManifest = $oStorageBackRest->manifest($strLibCSrcPath);
-
-                foreach my $strFile (sort(keys(%{$hManifest})))
-                {
-                    if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
-                    {
-                        $lTimestampLast = $hManifest->{$strFile}{modification_time};
-                    }
-                }
-            }
 
             # Rebuild if the modification time of the smart file does equal the last changes in source paths
             if ($bSmart)
             {
                 if (!$oStorageBackRest->exists($strLibCSmart) || $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
                 {
-                    &log(INFO, 'libc dependencies have changed, rebuilding...');
+                    &log(INFO, '    libc dependencies have changed, rebuilding...');
 
                     $bRebuild = true;
                 }
@@ -600,7 +595,7 @@ eval
 
                 if ($bRebuild)
                 {
-                    &log(INFO, "build C library for ${strBuildVM} (${strBuildPath})");
+                    &log(INFO, "    build C library for ${strBuildVM} (${strBuildPath})");
 
                     # It's very expensive to rebuild the Makefile so make sure it has actually changed
                     my $bMakeRebuild =