Added --log-timestamp option.

Allow timestamps to be suppressed in logging. This is primarily used to avoid filters in the automated documentation.

doc/lib/BackRestDoc/Common/DocExecute.pm

@@ -233,32 +233,13 @@ sub execute
 
                     # Trim off extra linefeeds before and after
                     $strOutput =~ s/^\n+|\n$//g;
-
-                    if ($strCommand =~ / pgbackrest /)
-                    {
-                        $strOutput =~ s/^                             //smg;
-                        $strOutput =~ s/^[0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{3} //smg;
-                    }
-
-                    # my @stryOutput = split("\n", $strOutput);
-                    # $$hCacheValue{stdout} = \@stryOutput;
                 }
 
                 if (defined($$hCacheKey{'err-expect'}) && defined($oExec->{strErrorLog}) && $oExec->{strErrorLog} ne '')
                 {
-                    # my $strError = $oExec->{strErrorLog};
-                    # $strError =~ s/^\n+|\n$//g;
-                    # my @stryError = split("\n", $strError);
-                    # $$hCacheValue{stderr} = \@stryError;
-
                     $strOutput .= $oExec->{strErrorLog};
                 }
 
-                # if (defined($$hCacheValue{stderr}))
-                # {
-                #     $strOutput .= join("\n", @{$$hCacheValue{stderr}});
-                # }
-
                 if ($$hCacheKey{output} && defined($$hCacheKey{highlight}) && $$hCacheKey{highlight}{filter} && defined($strOutput))
                 {
                     my $strHighLight = @{$$hCacheKey{highlight}{list}}[0];
@@ -583,6 +564,7 @@ sub backrestConfig
             # ??? This is not very pretty and should be replaced with a general way to hide config options
             my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
             delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_STDERR});
+            delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_TIMESTAMP});
 
             if (keys(%{$$oConfigClean{&CONFIG_SECTION_GLOBAL}}) == 0)
             {

doc/xml/reference.xml

@@ -68,6 +68,15 @@
 
                         <example>error</example>
                     </config-key>
+
+                    <!-- CONFIG - LOG SECTION - LOG-TIMESTAMP KEY -->
+                    <config-key id="log-timestamp" name="Log Timestamp">
+                        <summary>Enable timestamp in logging.</summary>
+
+                        <text>Enables the timestamp in console and file logging.  This option is disabled in special situations such as generating documentation.</text>
+
+                        <example>n</example>
+                    </config-key>
                 </config-key-list>
             </config-section>
 

doc/xml/release.xml

@@ -308,6 +308,10 @@
                     <release-item>
                         <p>Removed <br-option>--lock</br-option> option.  This option was introduced before the lock directory could be located outside the repository and is now obsolete.</p>
                     </release-item>
+
+                    <release-item>
+                        <p>Added <br-option>--log-timestamp</br-option> option to allow timestamps to be suppressed in logging.  This is primarily used to avoid filters in the automated documentation.</p>
+                    </release-item>
                 </release-refactor-list>
             </release-core-list>
 

doc/xml/user-guide.xml

@@ -330,7 +330,9 @@
                 <title>Configure the <postgres/> cluster data directory</title>
 
                 <backrest-config-option section="demo" key="db-path">{[db-path]}</backrest-config-option>
+
                 <backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
+                <backrest-config-option section="global" key="log-timestamp">n</backrest-config-option>
             </backrest-config>
 
             <p><backrest/> configuration files follow the Windows INI convention.  Sections are denoted by text in brackets and key/value pairs are contained in each section.  Lines beginning with <id>#</id> are ignored and can be used as comments.</p>
@@ -1350,6 +1352,7 @@
                 <backrest-config-option section="global" key="retention-full">2</backrest-config-option>
 
                 <backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
+                <backrest-config-option section="global" key="log-timestamp">n</backrest-config-option>
             </backrest-config>
 
             <p>The database host must be configured with the backup host/user.  The default for the <br-option>backup-user</br-option> option is <id>backrest</id>.  If the <id>postgres</id> user does restores on the backup host it is best not to also allow the <id>postgres</id> user to perform backups.  However, the <id>postgres</id> user can read the repository directly if it is in the same group as the <id>backrest</id> user.</p>
@@ -1367,6 +1370,7 @@
                 <backrest-config-option section="global" key="log-level-file">detail</backrest-config-option>
 
                 <backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
+                <backrest-config-option section="global" key="log-timestamp">n</backrest-config-option>
             </backrest-config>
 
             <p>The repository directory will also be removed from the database host.  It will not be used anymore so leaving it around may be confusing later on.</p>
@@ -1733,11 +1737,12 @@
                 <backrest-config-option section="demo" key="db-path">{[db-path]}</backrest-config-option>
 
                 <backrest-config-option section="global" key="repo-path">{[backrest-repo-path]}</backrest-config-option>
-
                 <backrest-config-option section="global" key="backup-host">{[host-backup]}</backrest-config-option>
-                <backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
 
                 <backrest-config-option section="demo" key="recovery-option">standby_mode=on</backrest-config-option>
+
+                <backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
+                <backrest-config-option section="global" key="log-timestamp">n</backrest-config-option>
             </backrest-config>
 
             <p>Now the standby can be created with the <cmd>restore</cmd> command.</p>

lib/pgBackRest/Common/Log.pm

@@ -71,6 +71,7 @@ my $strLogFileCache = undef;
 my $strLogLevelFile = OFF;
 my $strLogLevelConsole = OFF;
 my $strLogLevelStdErr = WARN;
+my $bLogTimestamp = true;
 
 # Flags to limit banner printing until there is actual output
 my $bLogFileExists;
@@ -166,6 +167,7 @@ sub logLevelSet
     my $strLevelFileParam = shift;
     my $strLevelConsoleParam = shift;
     my $strLevelStdErrParam = shift;
+    my $bLogTimestampParam = shift;
 
     # Load FileCommon module
     require pgBackRest::FileCommon;
@@ -200,6 +202,11 @@ sub logLevelSet
 
         $strLogLevelStdErr = uc($strLevelStdErrParam);
     }
+
+    if (defined($bLogTimestampParam))
+    {
+        $bLogTimestamp = $bLogTimestampParam;
+    }
 }
 
 push @EXPORT, qw(logLevelSet);
@@ -630,8 +637,8 @@ sub log
     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
 
     $strMessageFormat =
-        timestampFormat() . sprintf('.%03d P%02d', (gettimeofday() - int(gettimeofday())) * 1000,
+        ($bLogTimestamp ? timestampFormat() . sprintf('.%03d ', (gettimeofday() - int(gettimeofday())) * 1000) : '') .
-        (defined($iProcessId) ? $iProcessId : 0)) .
+        sprintf('P%02d', defined($iProcessId) ? $iProcessId : 0) .
         (' ' x (7 - length($strLevel))) . "${strLevel}: ${strMessageFormat}\n";
 
     # Skip output if disabled

lib/pgBackRest/Config/Config.pm

@@ -285,6 +285,8 @@ use constant OPTION_LOG_LEVEL_FILE                                  => 'log-leve
     push @EXPORT, qw(OPTION_LOG_LEVEL_FILE);
 use constant OPTION_LOG_LEVEL_STDERR                                => 'log-level-stderr';
     push @EXPORT, qw(OPTION_LOG_LEVEL_STDERR);
+use constant OPTION_LOG_TIMESTAMP                                   => 'log-timestamp';
+    push @EXPORT, qw(OPTION_LOG_TIMESTAMP);
 
 # ARCHIVE Section
 #-----------------------------------------------------------------------------------------------------------------------------------
@@ -483,6 +485,8 @@ use constant OPTION_DEFAULT_LOG_LEVEL_FILE                          => lc(INFO);
     push @EXPORT, qw(OPTION_DEFAULT_LOG_LEVEL_FILE);
 use constant OPTION_DEFAULT_LOG_LEVEL_STDERR                        => lc(WARN);
     push @EXPORT, qw(OPTION_DEFAULT_LOG_LEVEL_STDERR);
+use constant OPTION_DEFAULT_LOG_TIMESTAMP                           => true;
+    push @EXPORT, qw(OPTION_DEFAULT_LOG_TIMESTAMP);
 
 # ARCHIVE SECTION
 #-----------------------------------------------------------------------------------------------------------------------------------
@@ -1366,6 +1370,26 @@ my %oOptionRule =
         }
     },
 
+    &OPTION_LOG_TIMESTAMP =>
+    {
+        &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+        &OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN,
+        &OPTION_RULE_DEFAULT => OPTION_DEFAULT_LOG_TIMESTAMP,
+        &OPTION_RULE_COMMAND =>
+        {
+            &CMD_ARCHIVE_GET => true,
+            &CMD_ARCHIVE_PUSH => true,
+            &CMD_BACKUP => true,
+            &CMD_CHECK => true,
+            &CMD_EXPIRE => true,
+            &CMD_INFO => true,
+            &CMD_RESTORE => true,
+            &CMD_STANZA_CREATE => true,
+            &CMD_START => true,
+            &CMD_STOP => true
+        }
+    },
+
     # ARCHIVE Section
     #-------------------------------------------------------------------------------------------------------------------------------
     &OPTION_ARCHIVE_ASYNC =>
@@ -1912,7 +1936,8 @@ sub configLogging
         logLevelSet(
             optionValid(OPTION_LOG_LEVEL_FILE) ? optionGet(OPTION_LOG_LEVEL_FILE) : OFF,
             optionValid(OPTION_LOG_LEVEL_CONSOLE) ? optionGet(OPTION_LOG_LEVEL_CONSOLE) : OFF,
-            optionValid(OPTION_LOG_LEVEL_STDERR) ? optionGet(OPTION_LOG_LEVEL_STDERR) : OFF);
+            optionValid(OPTION_LOG_LEVEL_STDERR) ? optionGet(OPTION_LOG_LEVEL_STDERR) : OFF,
+            optionValid(OPTION_LOG_TIMESTAMP) ? optionGet(OPTION_LOG_TIMESTAMP) : undef);
 
         $bInitLog = true;
     }

lib/pgBackRest/Config/ConfigHelpData.pm

@@ -491,6 +491,18 @@ my $oConfigHelpData =
                     "log path is required."
         },
 
+        # LOG-TIMESTAMP Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'log-timestamp' =>
+        {
+            section => 'log',
+            summary =>
+                "Enable timestamp in logging.",
+            description =>
+                "Enables the timestamp in console and file logging. This option is disabled in special situations such as " .
+                    "generating documentation."
+        },
+
         # MANIFEST-SAVE-THRESHOLD Option Help
         #---------------------------------------------------------------------------------------------------------------------------
         'manifest-save-threshold' =>
@@ -832,6 +844,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'neutral-umask' => 'section',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
@@ -871,6 +884,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'neutral-umask' => 'section',
                 'process-max' => 'section',
                 'protocol-timeout' => 'section',
@@ -939,6 +953,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'manifest-save-threshold' => 'section',
                 'neutral-umask' => 'section',
                 'online' => 'default',
@@ -1018,6 +1033,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'neutral-umask' => 'section',
                 'online' => 'default',
                 'protocol-timeout' => 'section',
@@ -1050,6 +1066,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'repo-path' => 'section',
                 'retention-archive' => 'section',
                 'retention-archive-type' => 'section',
@@ -1103,6 +1120,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
 
                 # OUTPUT Option Help
                 #-------------------------------------------------------------------------------------------------------------------
@@ -1176,6 +1194,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'neutral-umask' => 'section',
                 'process-max' => 'section',
                 'protocol-timeout' => 'section',
@@ -1317,6 +1336,7 @@ my $oConfigHelpData =
                 'log-level-console' => 'section',
                 'log-level-file' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'neutral-umask' => 'section',
                 'online' => 'default',
                 'protocol-timeout' => 'section',
@@ -1353,6 +1373,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'repo-path' => 'section',
                 'stanza' => 'default'
             }
@@ -1401,6 +1422,7 @@ my $oConfigHelpData =
                 'log-level-file' => 'section',
                 'log-level-stderr' => 'section',
                 'log-path' => 'section',
+                'log-timestamp' => 'section',
                 'repo-path' => 'section',
                 'stanza' => 'default'
             }

test/expect/help-help-001.log

@@ -84,6 +84,7 @@ Log Options:
   --log-level-console       level for console logging [default=warn]
   --log-level-file          level for file logging [default=info]
   --log-level-stderr        level for stderr logging [default=warn]
+  --log-timestamp           enable timestamp in logging [default=y]
 
 Use 'pgbackrest help info [option]' for more information.
 

test/lib/pgBackRestTest/Common/LogTest.pm

@@ -114,15 +114,22 @@ sub logAdd
         # Do replacements on each line of the log
         foreach my $strLine (split("\n", $strLog))
         {
+            # Remove timestamp from log line
             $strLine =~ s/^[0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{3} //;
 
+            # Don't include test points in the final log
             if ($strLine !~ /^  TEST/)
             {
+                # Remove spaces that provided padding for the timetamp
                 $strLine =~ s/^                            //;
+
+                # Remove any final CRs
                 $strLine =~ s/\r$//;
 
+                # Do other replacements
                 $strLine = $self->regExpReplaceAll($strLine);
 
+                # Add to the log
                 $self->{strLog} .= "${strLine}\n";
             }
         }