Doc engine improvements.

Bug Fixes: * Fixed and issue that suppressed exceptions in PDF builds. Features: * Allow a source to be included as a section so large documents can be broken up. * Added section link support to Markdown output. * Added list support to PDF output. * Added include option to explicitly build sources (complements the exclude option though both cannot be used in the same invocation). * Added keyword-add option to add keywords without overriding the default keyword. * Added debug option to doc.pl to easily add the debug keyword to documentation builds. * Added pre option to doc.pl to easily add the pre keyword to documentation builds. Refactoring: * Improvements to markdown rendering. * Remove code dependency on project variable, instead use title param.
2025-10-30 23:37:45 +02:00 · 2016-11-17 16:35:11 -05:00
parent 54b1a784b5
commit c9b49b0d7e
16 changed files with 203 additions and 35 deletions
--- a/README.md
+++ b/README.md
@@ -73,7 +73,9 @@ pgBackRest includes support for versions down to 8.3, since older versions of Po
 pgBackRest strives to be easy to configure and operate:

 - [User guide](http://www.pgbackrest.org/user-guide.html) for Debian & Ubuntu / PostgreSQL 9.4.
+
 - [Command reference](http://www.pgbackrest.org/command.html) for command-line operations.
+
 - [Configuration reference](http://www.pgbackrest.org/configuration.html) for creating pgBackRest configurations.

 ## Contributions
--- a/doc/doc.pl
+++ b/doc/doc.pl
@@ -62,10 +62,16 @@ doc.pl [options]
   --var            Override variables defined in the XML
   --doc-path       Document path to render (manifest.xml should be located here)
   --out            Output types (html, pdf, markdown)
-   --keyword        Keyword used to filter output
-   --dev            Add dev keyword
   --require        Require only certain sections of the document (to speed testing)
+   --include        Include source in generation (links will reference website)
   --exclude        Exclude source from generation (links will reference website)
+
+Keyword Options:
+   --keyword        Keyword used to filter output
+   --keyword-add    Add keyword without overriding 'default' keyword
+   --dev            Add 'dev' keyword
+   --debug          Add 'debug' keyword
+   --pre            Add 'pre' keyword
 =cut

 ####################################################################################################################################
@@ -82,10 +88,14 @@ my $oVariableOverride = {};
 my $strDocPath;
 my @stryOutput;
 my @stryKeyword;
+my @stryKeywordAdd;
 my @stryRequire;
+my @stryInclude;
 my @stryExclude;
 my $bDeploy = false;
 my $bDev = false;
+my $bDebug = false;
+my $bPre = false;

 GetOptions ('help' => \$bHelp,
            'version' => \$bVersion,
@@ -93,12 +103,16 @@ GetOptions ('help' => \$bHelp,
            'log-level=s' => \$strLogLevel,
            'out=s@' => \@stryOutput,
            'keyword=s@' => \@stryKeyword,
+            'keyword-add=s@' => \@stryKeywordAdd,
            'require=s@' => \@stryRequire,
+            'include=s@' => \@stryInclude,
            'exclude=s@' => \@stryExclude,
            'no-exe', \$bNoExe,
            'deploy', \$bDeploy,
            'no-cache', \$bNoCache,
            'dev', \$bDev,
+            'debug', \$bDebug,
+            'pre', \$bPre,
            'cache-only', \$bCacheOnly,
            'var=s%', $oVariableOverride,
            'doc-path=s', \$strDocPath)
@@ -153,12 +167,33 @@ eval
        @stryKeyword = ('default');
    }

-    # If -dev passed then add the dev keyword
+    # Push added keywords
+    push(@stryKeyword, @stryKeywordAdd);
+
+    # If --dev passed then add the dev keyword
    if ($bDev)
    {
        push(@stryKeyword, 'dev');
    }

+    # If --debug passed then add the debug keyword
+    if ($bDebug)
+    {
+        push(@stryKeyword, 'debug');
+    }
+
+    # If --pre passed then add the pre keyword
+    if ($bPre)
+    {
+        push(@stryKeyword, 'pre');
+    }
+
+    # Doesn't make sense to pass include and exclude
+    if (@stryInclude > 0 && @stryExclude > 0)
+    {
+        confess "cannot specify both --include and --exclude";
+    }
+
    logLevelSet(undef, uc($strLogLevel), OFF);

    # Get the base path
@@ -180,7 +215,7 @@ eval

    # Load the manifest
    my $oManifest = new BackRestDoc::Common::DocManifest(
-        \@stryKeyword, \@stryRequire, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy, $bCacheOnly);
+        \@stryKeyword, \@stryRequire, \@stryInclude, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy, $bCacheOnly);

    if (!$bNoCache)
    {
--- a/doc/lib/BackRestDoc/Common/Doc.pm
+++ b/doc/lib/BackRestDoc/Common/Doc.pm
@@ -253,7 +253,7 @@ sub build
            {
                $$oOut{field}{text} = $oSub;
            }
-            elsif (defined($$oSub{value}) && !defined($$oSub{param}))
+            elsif ((defined($$oSub{value}) && !defined($$oSub{param})) && $strName ne 'code-block')
            {
                $$oOut{field}{$strName} = $$oSub{value};
            }
--- a/doc/lib/BackRestDoc/Common/DocManifest.pm
+++ b/doc/lib/BackRestDoc/Common/DocManifest.pm
@@ -58,6 +58,7 @@ sub new
        my $strOperation,
        $self->{stryKeyword},
        $self->{stryRequire},
+        $self->{stryInclude},
        $self->{stryExclude},
        my $oVariableOverride,
        $self->{strDocPath},
@@ -69,6 +70,7 @@ sub new
            __PACKAGE__ . '->new', \@_,
            {name => 'stryKeyword'},
            {name => 'stryRequire'},
+            {name => 'stryInclude'},
            {name => 'stryExclude'},
            {name => 'oVariableOverride', required => false},
            {name => 'strDocPath', required => false},
@@ -164,6 +166,12 @@ sub new
                next;
            }

+            # Skip sources not in include list
+            if (@{$self->{stryInclude}} > 0 && !grep(/^$strSource$/, @{$self->{stryInclude}}))
+            {
+                next;
+            }
+
            $$oRenderOutHash{source} = $strSource;

            # Get the filename
--- a/doc/lib/BackRestDoc/Common/DocRender.pm
+++ b/doc/lib/BackRestDoc/Common/DocRender.pm
@@ -43,6 +43,7 @@ my $oRenderTag =
        'cmd' => ['`', '`'],
        'param' => ['`', '`'],
        'setting' => ['`', '`'],
+        'pg-setting' => ['`', '`'],
        'code' => ['`', '`'],
        # 'code-block' => ['```', '```'],
        # 'exe' => [undef, ''],
@@ -299,6 +300,7 @@ sub build
    my $oNode = shift;
    my $oParent = shift;
    my $strPath = shift;
+    my $strPathPrefix = shift;

    # &log(INFO, "        node " . $oNode->nameGet());

@@ -326,8 +328,12 @@ sub build
            $self->{oSection} = {};
    }

+    # Build section
    if ($strName eq 'section')
    {
+        &log(DEBUG, 'build section [' . $oNode->paramGet('id') . ']');
+
+        # Set path and parent-path for this section
        if (defined($strPath))
        {
            $oNode->paramSet('path-parent', $strPath);
@@ -338,12 +344,55 @@ sub build
        &log(DEBUG, "            path ${strPath}");
        ${$self->{oSection}}{$strPath} = $oNode;
        $oNode->paramSet('path', $strPath);
+
+        # If section content is being pulled from elsewhere go get the content
+        if ($oNode->paramTest('source'))
+        {
+            my $oSource = ${$self->{oManifest}->sourceGet($oNode->paramGet('source'))}{doc};
+
+            foreach my $oSection ($oSource->nodeList('section'))
+            {
+                push(@{${$oNode->{oDoc}}{children}}, $oSection->{oDoc});
+            }
+
+            # Set path prefix to modify all section paths further down
+            $strPathPrefix = $strPath;
+        }
+    }
+    # Build link
+    elsif ($strName eq 'link')
+    {
+        &log(DEBUG, 'build link [' . $oNode->valueGet() . ']');
+
+        # If the path prefix is set and this is a section
+        if (defined($strPathPrefix) && $oNode->paramTest('section'))
+        {
+            my $strNewPath = $strPathPrefix . $oNode->paramGet('section');
+            &log(DEBUG, "modify link section from '" . $oNode->paramGet('section') . "' to '${strNewPath}'");
+
+            $oNode->paramSet('section', $strNewPath);
+        }
    }

-    # Iterate all nodes
+    # Iterate all text nodes
+    if (defined($oNode->textGet(false)))
+    {
+        foreach my $oChild ($oNode->textGet()->nodeList(undef, false))
+        {
+            if (ref(\$oChild) ne "SCALAR")
+            {
+                $self->build($oChild, $oNode, $strPath, $strPathPrefix);
+            }
+        }
+    }
+
+    # Iterate all non-text nodes
    foreach my $oChild ($oNode->nodeList(undef, false))
    {
-        $self->build($oChild, $oNode, $strPath);
+        if (ref(\$oChild) ne "SCALAR")
+        {
+            $self->build($oChild, $oNode, $strPath, $strPathPrefix);
+        }
    }
 }

@@ -496,6 +545,12 @@ sub processTag
            else
            {
                my $strSection = $oTag->paramGet('section');
+                my $oSection = ${$self->{oSection}}{$strSection};
+
+                if (!defined($oSection))
+                {
+                    confess &log(ERROR, "section link '${strSection}' does not exist");
+                }

                if (!defined($strSection))
                {
@@ -504,7 +559,7 @@ sub processTag

                if ($strType eq 'html')
                {
-                    $strUrl = '#' . substr($oTag->paramGet('section'), 1);
+                    $strUrl = '#' . substr($strSection, 1);
                }
                elsif ($strType eq 'latex')
                {
@@ -512,7 +567,10 @@ sub processTag
                }
                else
                {
-                    confess &log(ERROR, "section links not supported for type ${strType}, value '" . $oTag->valueGet() . "'");
+                    $strUrl = lc($self->processText($oSection->nodeGet('title')->textGet()));
+                    $strUrl =~ s/[^\w\- ]//g;
+                    $strUrl =~ s/ /-/g;
+                    $strUrl = '#' . $strUrl;
                }
            }
        }
--- a/doc/lib/BackRestDoc/Html/DocHtmlPage.pm
+++ b/doc/lib/BackRestDoc/Html/DocHtmlPage.pm
@@ -85,8 +85,7 @@ sub process
    my $oPage = $self->{oDoc};

    # Initialize page
-    my $strTitle = "{[project]}" .
-                   (defined($oPage->paramGet('title', false)) ? ' ' . $oPage->paramGet('title') : '');
+    my $strTitle = $oPage->paramGet('title');
    my $strSubTitle = $oPage->paramGet('subtitle', false);

    my $oHtmlBuilder = new BackRestDoc::Html::DocHtmlBuilder(
--- a/doc/lib/BackRestDoc/Latex/DocLatex.pm
+++ b/doc/lib/BackRestDoc/Latex/DocLatex.pm
@@ -100,6 +100,9 @@ sub process

    my $strLatex = $self->{oManifest}->variableReplace(fileStringRead($self->{strPreambleFile}), 'latex') . "\n";

+    # !!! Temp hack for underscores in filename
+    $strLatex =~ s/pgaudit\\\_doc/pgaudit\_doc/g;
+
    foreach my $strPageId ($self->{oManifest}->renderOutList(RENDER_TYPE_PDF))
    {
        &log(INFO, "    render out: ${strPageId}");
@@ -129,6 +132,10 @@ sub process
                # Save the html page
                $strLatex .= $oDocLatexSection->process();
            }
+            else
+            {
+                confess $oException;
+            }
        };
    }

--- a/doc/lib/BackRestDoc/Latex/DocLatexSection.pm
+++ b/doc/lib/BackRestDoc/Latex/DocLatexSection.pm
@@ -71,8 +71,7 @@ sub process
    my $strLatex;

    # Initialize page
-    my $strTitle = "{[project]}" .
-                   (defined($oPage->paramGet('title', false)) ? ' ' . $oPage->paramGet('title') : '');
+    my $strTitle = $oPage->paramGet('title');
    my $strSubTitle = $oPage->paramGet('subtitle', false);

    # Render sections
@@ -325,6 +324,18 @@ sub sectionProcess

            $strLatex .= "\n" . $self->processText($oDescription) . "\n";
        }
+        # Add a list
+        elsif ($oChild->nameGet() eq 'list')
+        {
+            $strLatex .= "\n\\begin{itemize}";
+
+            foreach my $oListItem ($oChild->nodeList())
+            {
+                $strLatex .= "\n    \\item " . $self->processText($oListItem->textGet());
+            }
+
+            $strLatex .= "\n\\end{itemize}";
+        }
        # Add/remove config options
        elsif ($oChild->nameGet() eq 'backrest-config' || $oChild->nameGet() eq 'postgres-config')
        {
--- a/doc/lib/BackRestDoc/Markdown/DocMarkdownRender.pm
+++ b/doc/lib/BackRestDoc/Markdown/DocMarkdownRender.pm
@@ -72,8 +72,7 @@ sub process
    my $oPage = $self->{oDoc};

    # Initialize page
-    my $strMarkdown = "# {[project]}" .
-                   (defined($oPage->paramGet('title', false)) ? ' ' . $oPage->paramGet('title') : '');
+    my $strMarkdown = "# " . $oPage->paramGet('title');

    if (defined($oPage->paramGet('subtitle', false)))
    {
@@ -280,7 +279,12 @@ sub sectionProcess
        {
            if ($oChild->paramTest('title'))
            {
-                $strMarkdown .= "\n\n_" . $oChild->paramGet('title') . "_:";
+                if (defined($strLastChild) && $strLastChild ne 'code-block')
+                {
+                    $strMarkdown .= "\n";
+                }
+
+                $strMarkdown .= "\n_" . $oChild->paramGet('title') . "_:";
            }

            $strMarkdown .= "\n```\n" . trim($oChild->valueGet()) . "\n```";
@@ -333,11 +337,9 @@ sub sectionProcess
        # Add a list
        elsif ($oChild->nameGet() eq 'list')
        {
-            $strMarkdown .= "\n";
-
            foreach my $oListItem ($oChild->nodeList())
            {
-                $strMarkdown .= "\n- " . $self->processText($oListItem->textGet());
+                $strMarkdown .= "\n\n- " . $self->processText($oListItem->textGet());
            }
        }
        # Add a subsection
--- a/doc/xml/dtd/doc.dtd
+++ b/doc/xml/dtd/doc.dtd
@@ -1,6 +1,6 @@
 <!ELEMENT doc ((description, intro, contributor-list, release-list)|(config, operation)|
-               (description, variable-list?, cleanup?, section+))>
-    <!ATTLIST doc title CDATA "">
+               (description?, variable-list?, cleanup?, section+))>
+    <!ATTLIST doc title CDATA #REQUIRED>
    <!ATTLIST doc subtitle CDATA "">
    <!ATTLIST doc toc CDATA "y">
    <!ATTLIST doc toc-number CDATA "y">
@@ -111,13 +111,14 @@
 <!ELEMENT cmd-description EMPTY>
    <!ATTLIST cmd-description key CDATA #REQUIRED>

-<!ELEMENT section (title,
+<!ELEMENT section (title?,
                   ((p|list|table|host-add|execute-list|backrest-config|postgres-config|cmd-description|
                     option-description|code-block)+|
                   ((p|list)*, section+)|(p|list)*))>
    <!ATTLIST section id CDATA #REQUIRED>
    <!ATTLIST section keyword CDATA "">
    <!ATTLIST section depend CDATA "">
+    <!ATTLIST section source CDATA "">
 <!ELEMENT title (#PCDATA|quote|b|i|bi|ul|ol|id|code|host|file|path|cmd|setting|exe|backrest|postgres|br-option|br-setting|
                 pg-option|pg-setting|link|user)*>

@@ -212,7 +213,7 @@
 <!ELEMENT id (#PCDATA)>
 <!ELEMENT code (#PCDATA)>
 <!ELEMENT code-block (#PCDATA|exe)*>
-    <!ATTLIST code-block title CDATA #REQUIRED>
+    <!ATTLIST code-block title CDATA "">

 <!ELEMENT host (#PCDATA)>
 <!ELEMENT file (#PCDATA)>
--- a/doc/xml/index.xml
+++ b/doc/xml/index.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc subtitle="Reliable {[postgres]} Backup &amp; Restore" toc="n">
+<doc title="{[project]}" subtitle="Reliable {[postgres]} Backup &amp; Restore" toc="n">
    <description>{[project]} provides fast, reliable backup and restore for {[postgres]} and seamlessly scales to terabyte scale databases by implementing stream compression and multi-processing.</description>

    <variable-list>
--- a/doc/xml/reference.xml
+++ b/doc/xml/reference.xml
@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc title="Command &amp; Configuration Reference">
+<doc title="{[project]} Command &amp; Configuration Reference">
    <!-- CONFIG -->
-    <config title="Configuration Reference">
+    <config title="{[project]} Configuration Reference">
        <description>The {[project]} Configuration Reference details all configuration options.</description>

        <text><backrest/> can be used entirely with command-line parameters but a configuration file is more practical for installations that are complex or set a lot of options. The default location for the configuration file is <file>/etc/pgbackrest.conf</file>.</text>
@@ -567,7 +567,7 @@
    </config>

    <!-- COMMAND -->
-    <operation title="Command Reference">
+    <operation title="{[project]} Command Reference">
        <description>The {[project]} Command Reference details all commands and options.</description>

        <text>Commands are used to execute the various <backrest/> functions.  Here the command options are listed exhaustively, that is, each option applicable to a command is listed with that command even if it applies to one or more other commands.  This includes all the options that may also configured in <file>pgbackrest.conf</file>.</text>
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc title="Releases" toc-number="n">
+<doc title="{[project]} Releases" toc-number="n">
    <description>The {[project]} Releases detail each version of the software and lists the changes made in each version.</description>

    <intro>
@@ -133,6 +133,56 @@
    </contributor-list>

    <release-list>
+        <release date="XXXX-XX-XX" version="1.12dev" title="UNDER DEVELOPMENT">
+            <release-doc-list>
+                <release-bug-list>
+                    <release-item>
+                        <p>Fixed and issue that suppressed exceptions in PDF builds.</p>
+                    </release-item>
+                </release-bug-list>
+
+                <release-feature-list>
+                    <release-item>
+                        <p>Allow a source to be included as a section so large documents can be broken up.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added section link support to Markdown output.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added list support to PDF output.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added <setting>include</setting> option to explicitly build sources (complements the <setting>exclude</setting> option though both cannot be used in the same invocation).</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added <setting>keyword-add</setting> option to add keywords without overriding the <id>default</id> keyword.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added <setting>debug</setting> option to <file>doc.pl</file> to easily add the <id>debug</id> keyword to documentation builds.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Added <setting>pre</setting> option to <file>doc.pl</file> to easily add the <id>pre</id> keyword to documentation builds.</p>
+                    </release-item>
+                </release-feature-list>
+
+                <release-refactor-list>
+                    <release-item>
+                        <p>Improvements to markdown rendering.</p>
+                    </release-item>
+
+                    <release-item>
+                        <p>Remove code dependency on <id>project</id> variable, instead use <id>title</id> param.</p>
+                    </release-item>
+                </release-refactor-list>
+            </release-doc-list>
+        </release>
+
        <release date="2016-11-17" version="1.11" title="Bug Fix for Asynchronous Archiving Efficiency">
            <release-core-list>
                <release-bug-list>
--- a/doc/xml/test.xml
+++ b/doc/xml/test.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc subtitle="Regression, Unit, &amp; Integration Testing" toc="y">
+<doc title="{[project]}" subtitle="Regression, Unit, &amp; Integration Testing" toc="y">
    <description>{[project]} testing configuration requirements, setup, and options.</description>

    <section id="introduction">
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc title="User Guide" subtitle="{[user-guide-subtitle]} / {[postgres]} {[pg-version]}">
+<doc title="{[project]} User Guide" subtitle="{[user-guide-subtitle]} / {[postgres]} {[pg-version]}">
    <description>The {[project]} User Guide demonstrates how to quickly and easily setup {[project]} for your {[postgres]} database. Step-by-step instructions lead the user through all the important features of the fastest, most reliable {[postgres]} backup and restore solution.</description>

    <variable-list>
--- a/test/README.md
+++ b/test/README.md
@@ -24,27 +24,22 @@ _Run All Tests_:
 ```
 /backrest/test/test.pl
 ```
-
 _Run Tests for a Specific OS_:
 ```
 /backrest/test/test.pl --vm=co6
 ```
-
 _Run Tests for a Specific OS and Module_:
 ```
 /backrest/test/test.pl --vm=co6 --module=backup
 ```
-
 _Run Tests for a Specific OS, Module, and Test_:
 ```
 /backrest/test/test.pl --vm=co6 --module=backup --test=full
 ```
-
 _Run Tests for a Specific OS, Module, Test, and Run_:
 ```
 /backrest/test/test.pl --vm=co6 --module=backup --test=full --run=1
 ```
-
 _Run Tests for a Specific OS, Module, Test, and Process Max_:
 ```
 /backrest/test/test.pl --vm=co6 --module=backup --test=full --process-max=4