Use the git log to ease release note management.

The release notes are generally a direct reflection of the git log.  So, ease the burden of maintaining the release notes by using the git log to determine what needs to be added.

Currently only non-dev items are required to be matched to a git commit but the goal is to account for all commits.

The git history cache is generated from the git log but can be modified to correct typos and match the release notes as they evolve.  The commit hash is used to identify commits that have already been added to the cache.

There's plenty more to do here.  For instance, links to the commits for each release item should be added to the release notes.

doc/lib/BackRestDoc/Common/DocRender.pm

@@ -167,8 +167,8 @@ sub new
         (
             __PACKAGE__ . '->new', \@_,
             {name => 'strType'},
-            {name => 'oManifest'},
+            {name => 'oManifest', required => false},
-            {name => 'bExe'},
+            {name => 'bExe', required => false},
             {name => 'strRenderOutKey', required => false}
         );
 
@@ -275,6 +275,20 @@ sub new
     );
 }
 
+####################################################################################################################################
+# Set begin and end values for a tag
+####################################################################################################################################
+sub tagSet
+{
+    my $self = shift;
+    my $strTag = shift;
+    my $strBegin = shift;
+    my $strEnd = shift;
+
+    $oRenderTag->{$self->{strType}}{$strTag}[0] = defined($strBegin) ? $strBegin : '';
+    $oRenderTag->{$self->{strType}}{$strTag}[1] = defined($strEnd) ? $strEnd : '';
+}
+
 ####################################################################################################################################
 # variableReplace
 #
@@ -284,7 +298,7 @@ sub variableReplace
 {
     my $self = shift;
 
-    return $self->{oManifest}->variableReplace(shift, $self->{strType});
+    return defined($self->{oManifest}) ? $self->{oManifest}->variableReplace(shift, $self->{strType}) : shift;
 }
 
 ####################################################################################################################################

doc/lib/BackRestDoc/Custom/DocCustomRelease.pm

@@ -7,11 +7,15 @@ use strict;
 use warnings FATAL => qw(all);
 use Carp qw(confess);
 
+use Cwd qw(abs_path);
 use Exporter qw(import);
     our @EXPORT = qw();
+use File::Basename qw(dirname);
 
 use pgBackRest::Common::Log;
 use pgBackRest::Common::String;
+use pgBackRest::Version;
+
 use pgBackRestBuild::Config::Data;
 
 use BackRestDoc::Common::DocRender;
@@ -245,6 +249,60 @@ sub contributorTextGet
     return $strContributorText;
 }
 
+####################################################################################################################################
+# Find a commit by subject prefix.  Error if the prefix appears more than once.
+####################################################################################################################################
+sub commitFindSubject
+{
+    my $self = shift;
+    my $rhyCommit = shift;
+    my $strSubjectPrefix = shift;
+    my $bRegExp = shift;
+
+    $bRegExp = defined($bRegExp) ? $bRegExp : true;
+    my $rhResult = undef;
+
+    foreach my $rhCommit (@{$rhyCommit})
+    {
+        if (($bRegExp && $rhCommit->{subject} =~ /^$strSubjectPrefix/) ||
+            (!$bRegExp && length($rhCommit->{subject}) >= length($strSubjectPrefix) &&
+                substr($rhCommit->{subject}, 0, length($strSubjectPrefix)) eq $strSubjectPrefix))
+        {
+            if (defined($rhResult))
+            {
+                confess &log(ERROR, "subject prefix '${strSubjectPrefix}' already found in commit " . $rhCommit->{commit});
+            }
+
+            $rhResult = $rhCommit;
+        }
+    }
+
+    return $rhResult;
+}
+
+####################################################################################################################################
+# Throw an error that includes a list of release commits
+####################################################################################################################################
+sub commitError
+{
+    my $self = shift;
+    my $strMessage = shift;
+    my $rstryCommitRemaining = shift;
+    my $rhyCommit = shift;
+
+    my $strList;
+
+    foreach my $strCommit (@{$rstryCommitRemaining})
+    {
+        $strList .=
+            (defined($strList) ? "\n" : '') .
+            substr($rhyCommit->{$strCommit}{date}, 0, length($rhyCommit->{$strCommit}{date}) - 15) . " $strCommit: " .
+            $rhyCommit->{$strCommit}{subject};
+    }
+
+    confess &log(ERROR, "${strMessage}:\n${strList}");
+}
+
 ####################################################################################################################################
 # docGet
 #
@@ -257,6 +315,15 @@ sub docGet
     # Assign function parameters, defaults, and log debug info
     my $strOperation = logDebugParam(__PACKAGE__ . '->docGet');
 
+    # Load the git history
+    my $oStorageDoc = new pgBackRest::Storage::Local(
+        dirname(abs_path($0)), new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false}));
+    my @hyGitLog = @{(JSON::PP->new()->allow_nonref())->decode(${$oStorageDoc->get("resource/git-history.cache")})};
+
+    # Get renderer
+    my $oRender = new BackRestDoc::Common::DocRender('text');
+    $oRender->tagSet('backrest', PROJECT_NAME);
+
     # Create the doc
     my $oDoc = new BackRestDoc::Common::Doc();
     $oDoc->paramSet('title', $self->{oDoc}->paramGet('title'));
@@ -277,16 +344,85 @@ sub docGet
     my $iStableReleaseTotal = 0;
     my $iUnsupportedReleaseTotal = 0;
 
-    foreach my $oRelease ($self->{oDoc}->nodeGet('release-list')->nodeList('release'))
+    my @oyRelease = $self->{oDoc}->nodeGet('release-list')->nodeList('release');
+
+    for (my $iReleaseIdx = 0; $iReleaseIdx < @oyRelease; $iReleaseIdx++)
     {
-        # Get the release version
+        my $oRelease = $oyRelease[$iReleaseIdx];
+
+        # Get the release version and dev flag
         my $strVersion = $oRelease->paramGet('version');
+        my $bReleaseDev = $strVersion =~ /dev$/ ? true : false;
+
+        # Get a list of commits that apply to this release
+        my @rhyReleaseCommit;
+        my $rhReleaseCommitRemaining;
+        my @stryReleaseCommitRemaining;
+        my $bReleaseCheckCommit = false;
+
+        if ($strVersion ge '2.01')
+        {
+            # Should commits in the release be checked?
+            $bReleaseCheckCommit = !$bReleaseDev ? true : false;
+
+            # Get the begin commit
+            my $rhReleaseCommitBegin = $self->commitFindSubject(\@hyGitLog, "Begin v${strVersion} development\\.");
+            my $strReleaseCommitBegin = defined($rhReleaseCommitBegin) ? $rhReleaseCommitBegin->{commit} : undef;
+
+            # Get the end commit of the last release
+            my $strReleaseLastVersion = $oyRelease[$iReleaseIdx + 1]->paramGet('version');
+            my $rhReleaseLastCommitEnd =  $self->commitFindSubject(\@hyGitLog, "v${strReleaseLastVersion}\\: .+");
+
+            if (!defined($rhReleaseLastCommitEnd))
+            {
+                confess &log(ERROR, "release ${strReleaseLastVersion} must have an end commit");
+            }
+
+            my $strReleaseLastCommitEnd = $rhReleaseLastCommitEnd->{commit};
+
+            # Get the end commit
+            my $rhReleaseCommitEnd = $self->commitFindSubject(\@hyGitLog, "v${strVersion}\\: .+");
+            my $strReleaseCommitEnd = defined($rhReleaseCommitEnd) ? $rhReleaseCommitEnd->{commit} : undef;
+
+            if ($bReleaseCheckCommit && !defined($rhReleaseCommitEnd) && $iReleaseIdx != 0)
+            {
+                confess &log(ERROR, "release ${strVersion} must have an end commit");
+            }
+
+            # Make a list of commits for this release
+            while ($hyGitLog[0]->{commit} ne $strReleaseLastCommitEnd)
+            {
+                # Don't add begin/end commits to the list since they are already accounted for
+                if ((defined($strReleaseCommitEnd) && $hyGitLog[0]->{commit} eq $strReleaseCommitEnd) ||
+                    (defined($strReleaseCommitBegin) && $hyGitLog[0]->{commit} eq $strReleaseCommitBegin))
+                {
+                    shift(@hyGitLog);
+                }
+                # Else add the commit to this releases' list
+                else
+                {
+                    push(@stryReleaseCommitRemaining, $hyGitLog[0]->{commit});
+                    push(@rhyReleaseCommit, $hyGitLog[0]);
+
+                    $rhReleaseCommitRemaining->{$hyGitLog[0]->{commit}}{date} = $hyGitLog[0]->{date};
+                    $rhReleaseCommitRemaining->{$hyGitLog[0]->{commit}}{subject} = $hyGitLog[0]->{subject};
+
+                    shift(@hyGitLog);
+                }
+            }
+
+            # At least one commit is required for non-dev releases
+            if ($bReleaseCheckCommit && @stryReleaseCommitRemaining == 0)
+            {
+                confess &log(ERROR, "no commits found for release ${strVersion}");
+            }
+        }
 
         # Display versions in TOC?
         my $bTOC = true;
 
         # Create a release section
-        if ($strVersion =~ /dev$/)
+        if ($bReleaseDev)
         {
             if ($iDevReleaseTotal > 1)
             {
@@ -354,7 +490,7 @@ sub docGet
         $oReleaseSection->paramSet(XML_SECTION_PARAM_ANCHOR, XML_SECTION_PARAM_ANCHOR_VALUE_NOINHERIT);
 
         $oReleaseSection->nodeAdd('title')->textSet(
-            "v${strVersion} " . ($strVersion =~ /dev$/ ? '' : 'Release ') . 'Notes');
+            "v${strVersion} " . ($bReleaseDev ? '' : 'Release ') . 'Notes');
 
         $oReleaseSection->nodeAdd('subtitle')->textSet($oRelease->paramGet('title'));
         $oReleaseSection->nodeAdd('subsubtitle')->textSet($strDateOut);
@@ -420,6 +556,49 @@ sub docGet
                             my @rhyReleaseItemP = $oReleaseFeature->nodeList('p');
                             my $oReleaseItemText = $rhyReleaseItemP[0]->textGet();
 
+                            # Check release item commits
+                            if ($bReleaseCheckCommit && $strItemType ne XML_RELEASE_DEVELOPMENT_LIST)
+                            {
+                                my @oyCommit = $oReleaseFeature->nodeList('commit', false);
+
+                                # If no commits found then try to use the description as the commit subject
+                                if (@oyCommit == 0)
+                                {
+                                    my $strSubject = $oRender->processText($oReleaseItemText);
+                                    my $rhCommit = $self->commitFindSubject(\@rhyReleaseCommit, $strSubject, false);
+
+                                    if (!defined($rhCommit))
+                                    {
+                                        $self->commitError(
+                                            "unable to find commit or no subject match for release ${strVersion} item" .
+                                                " '${strSubject}'",
+                                            \@stryReleaseCommitRemaining, $rhReleaseCommitRemaining);
+
+                                        my $strCommit = $rhCommit->{commit};
+                                        @stryReleaseCommitRemaining = grep(!/$strCommit/, @stryReleaseCommitRemaining);
+                                    }
+                                }
+
+                                # Check the rest of the commits to ensure they exist
+                                foreach my $oCommit (@oyCommit)
+                                {
+                                    my $strSubject = $oCommit->paramGet('subject');
+                                    my $rhCommit = $self->commitFindSubject(\@rhyReleaseCommit, $strSubject, false);
+
+                                    if (defined($rhCommit))
+                                    {
+                                        my $strCommit = $rhCommit->{commit};
+                                        @stryReleaseCommitRemaining = grep(!/$strCommit/, @stryReleaseCommitRemaining);
+                                    }
+                                    else
+                                    {
+                                        $self->commitError(
+                                            "unable to find release ${strVersion} commit subject '${strSubject}' in list",
+                                            \@stryReleaseCommitRemaining, $rhReleaseCommitRemaining);
+                                    }
+                                }
+                            }
+
                             # Append the rest of the text
                             if (@rhyReleaseItemP > 1)
                             {
@@ -451,6 +630,13 @@ sub docGet
                 }
             }
         }
+
+        # Error if there are commits left over
+        # if ($bReleaseCheckCommit && @stryReleaseCommitRemaining != 0)
+        # {
+        #     $self->commitError(
+        #         "unassigned commits for release ${strVersion}", \@stryReleaseCommitRemaining, $rhReleaseCommitRemaining);
+        # }
     }
 
     # Return from function and log return values if any

doc/release.pl

@@ -62,7 +62,7 @@ release.pl [options]
  Release Options:
    --build          Build the cache before release (should be included in the release commit)
    --deploy         Deploy documentation to website (can be done as docs are updated)
-   --no-coverage    Don't generate the coverage report
+   --no-gen         Don't auto-generate
 =cut
 
 ####################################################################################################################################
@@ -74,7 +74,7 @@ my $bQuiet = false;
 my $strLogLevel = 'info';
 my $bBuild = false;
 my $bDeploy = false;
-my $bNoCoverage = false;
+my $bNoGen = false;
 
 GetOptions ('help' => \$bHelp,
             'version' => \$bVersion,
@@ -82,7 +82,7 @@ GetOptions ('help' => \$bHelp,
             'log-level=s' => \$strLogLevel,
             'build' => \$bBuild,
             'deploy' => \$bDeploy,
-            'no-coverage' => \$bNoCoverage)
+            'no-gen' => \$bNoGen)
     or pod2usage(2);
 
 ####################################################################################################################################
@@ -133,21 +133,94 @@ eval
 
     if ($bBuild)
     {
-        # Remove permanent cache file
+        if (!$bNoGen)
-        $oStorageDoc->remove("${strDocPath}/resource/exe.cache", {bIgnoreMissing => true});
-
-        # Remove all docker containers to get consistent IP address assignments
-        executeTest('docker rm -f $(docker ps -a -q)', {bSuppressError => true});
-
-        # Generate coverage summmary
-        if (!$bNoCoverage)
         {
+            # Update git history
+            my $strGitCommand =
+                'git -C ' . $strDocPath .
+                ' log --pretty=format:\'{^^^^commit^^^^:^^^^%H^^^^,^^^^date^^^^:^^^^%ci^^^^,^^^^subject^^^^:^^^^%s^^^^,^^^^body^^^^:^^^^%b^^^^},\'';
+            my $strGitLog = qx($strGitCommand);
+            $strGitLog =~ s/\^\^\^\^\}\,\n/\#\#\#\#/mg;
+            $strGitLog =~ s/\\/\\\\/g;
+            $strGitLog =~ s/\n/\\n/mg;
+            $strGitLog =~ s/\r/\\r/mg;
+            $strGitLog =~ s/\t/\\t/mg;
+            $strGitLog =~ s/\"/\\\"/g;
+            $strGitLog =~ s/\^\^\^\^/\"/g;
+            $strGitLog =~ s/\#\#\#\#/\"\}\,\n/mg;
+            $strGitLog = '[' . substr($strGitLog, 0, length($strGitLog) - 1) . ']';
+            my @hyGitLog = @{(JSON::PP->new()->allow_nonref())->decode($strGitLog)};
+
+            # Load prior history
+            my @hyGitLogPrior = @{(JSON::PP->new()->allow_nonref())->decode(
+                ${$oStorageDoc->get("${strDocPath}/resource/git-history.cache")})};
+
+            # Add new commits
+            for (my $iGitLogIdx = @hyGitLog - 1; $iGitLogIdx >= 0; $iGitLogIdx--)
+            {
+                my $rhGitLog = $hyGitLog[$iGitLogIdx];
+                my $bFound = false;
+
+                foreach my $rhGitLogPrior (@hyGitLogPrior)
+                {
+                    if ($rhGitLog->{commit} eq $rhGitLogPrior->{commit})
+                    {
+                        $bFound = true;
+                    }
+                }
+
+                next if $bFound;
+
+                $rhGitLog->{body} = trim($rhGitLog->{body});
+
+                if ($rhGitLog->{body} eq '')
+                {
+                    delete($rhGitLog->{body});
+                }
+
+                unshift(@hyGitLogPrior, $rhGitLog);
+            }
+
+            # Write git log
+            $strGitLog = undef;
+
+            foreach my $rhGitLog (@hyGitLogPrior)
+            {
+                $strGitLog .=
+                    (defined($strGitLog) ? ",\n" : '') .
+                    "    {\n" .
+                    '        "commit": ' . trim((JSON::PP->new()->allow_nonref()->pretty())->encode($rhGitLog->{commit})) . ",\n" .
+                    '        "date": ' . trim((JSON::PP->new()->allow_nonref()->pretty())->encode($rhGitLog->{date})) . ",\n" .
+                    '        "subject": ' . trim((JSON::PP->new()->allow_nonref()->pretty())->encode($rhGitLog->{subject}));
+
+                # Skip the body if it is empty or a release (since we already have the release note content)
+                if ($rhGitLog->{subject} !~ /^v[0-9]{1,2}\.[0-9]{1,2}\: /g && defined($rhGitLog->{body}))
+                {
+                    $strGitLog .=
+                        ",\n" .
+                        '        "body": ' . trim((JSON::PP->new()->allow_nonref()->pretty())->encode($rhGitLog->{body}));
+                }
+
+                $strGitLog .=
+                    "\n" .
+                    "    }";
+            }
+
+            $oStorageDoc->put("${strDocPath}/resource/git-history.cache", "[\n${strGitLog}\n]\n");
+
+            # Generate coverage summmary
             &log(INFO, "Generate Coverage Summary");
             executeTest(
                 "${strTestExe} --no-lint --no-package --no-valgrind --no-optimize --vm-max=3 --coverage-summary",
                 {bShowOutputAsync => true});
         }
 
+        # Remove permanent cache file
+        $oStorageDoc->remove("${strDocPath}/resource/exe.cache", {bIgnoreMissing => true});
+
+        # Remove all docker containers to get consistent IP address assignments
+        executeTest('docker rm -f $(docker ps -a -q)', {bSuppressError => true});
+
         # Generate deployment docs for RHEL/Centos 7
         &log(INFO, "Generate RHEL/CentOS 7 documentation");
 

doc/xml/dtd/doc.dtd

@@ -168,7 +168,7 @@
 <!ELEMENT release-improvement-list (release-item+)>
 <!ELEMENT release-development-list (release-item+)>
 
-<!ELEMENT release-item (release-item-contributor-list?, p+)>
+<!ELEMENT release-item (commit*, release-item-contributor-list?, p+)>
 
 <!ELEMENT release-item-contributor-list (release-item-ideator*, release-item-contributor*, release-item-reviewer*)>
 
@@ -179,6 +179,11 @@
 <!ELEMENT release-item-reviewer (#PCDATA)>
     <!ATTLIST release-item-reviewer id CDATA #REQUIRED>
 
+<!ELEMENT commit EMPTY>
+    <!ATTLIST commit type CDATA "">
+    <!ATTLIST commit id CDATA "">
+    <!ATTLIST commit subject CDATA #REQUIRED>
+
 <!ELEMENT contribute (text)>
     <!ATTLIST contribute title CDATA #REQUIRED>
 

doc/xml/release.xml

@@ -386,6 +386,8 @@
                     </release-item>
 
                     <release-item>
+                        <!-- <commit type="depend" subject="Allow three-digits process IDs in logging."/> -->
+
                         <release-item-contributor-list>
                             <release-item-ideator id="rakshitha.br"/>
                         </release-item-contributor-list>
@@ -846,6 +848,9 @@
                     </release-item>
 
                     <release-item>
+                        <commit subject="Include Posix-compliant header for strcasecmp()."/>
+                        <commit subject="Include Posix-compliant header for fd_set."/>
+
                         <release-item-contributor-list>
                             <release-item-ideator id="ucando"/>
                         </release-item-contributor-list>
@@ -966,6 +971,9 @@
                     </release-item>
 
                     <release-item>
+                        <commit subject="Allow if in manifest variables."/>
+                        <commit subject="Allow if condition in documentation lists and list items."/>
+
                         <p>Allow <code>if</code> in manifest variables, lists, and list items.</p>
                     </release-item>
                 </release-improvement-list>
@@ -1577,6 +1585,10 @@
                     </release-item>
 
                     <release-item>
+                        <commit subject="PostgreSQL 11 Beta 4 support."/>
+                        <commit subject="Support configurable WAL segment size."/>
+                        <commit subject="PostgreSQL 11 support."/>
+
                         <p><postgres/> 11 support, including configurable WAL segment size.</p>
                     </release-item>
                 </release-feature-list>

test/travis.pl

@@ -119,7 +119,7 @@ eval
         processEnd();
 
         processBegin('release documentation doc');
-        executeTest("${strReleaseExe} --build --no-coverage", {bShowOutputAsync => true});
+        executeTest("${strReleaseExe} --build --no-gen", {bShowOutputAsync => true});
         processEnd();
     }