Build command and configuration reference in C.

Migrate generation of these files from help.xml to the intermediate documentation format. This allows us to share a lot of code that is already in C and remove duplicated code in Perl. More duplicate code can be removed in Perl once man generation is migrated.

Also update the unit test harness to allow testing of modules in the doc directory.

test/ci.pl

@@ -131,7 +131,7 @@ eval
     processBegin('install common packages');
     processExec('sudo apt-get -qq update', {bSuppressStdErr => true, bSuppressError => true});
     processExec(
-        'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libxml-checker-perl libyaml-perl', {bSuppressStdErr => true});
+        'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y meson libxml-checker-perl libyaml-perl', {bSuppressStdErr => true});
     processEnd();
 
     if (!$bNoTempFs)
@@ -182,7 +182,7 @@ eval
     {
         # Build list of packages that need to be installed
         my $strPackage =
-            "make gcc ccache meson python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" .
+            "make gcc ccache python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" .
             " uncrustify libssh2-1-dev";
 
         # Add lcov when testing coverage

test/define.yaml

@@ -973,6 +973,18 @@ unit:
           - command/server/ping
           - command/server/server
 
+  # ********************************************************************************************************************************
+  - name: doc
+
+    test:
+      # ----------------------------------------------------------------------------------------------------------------------------
+      - name: build
+        total: 1
+
+        coverage:
+          - doc/command/build/build
+          - doc/command/build/reference
+
 # **********************************************************************************************************************************
 # Integration tests
 #

test/lib/pgBackRestTest/Common/CoverageTest.pm

@@ -129,6 +129,10 @@ sub coverageExtract
         {
             $strModuleName =~ s/^test/src/mg;
         }
+        elsif ($strModuleName =~ /^doc/mg)
+        {
+            $strModuleName =~ s/^doc/doc\/src/mg;
+        }
 
         my $strModuleOutName = $strModuleName;
         my $bTest = false;
@@ -150,6 +154,10 @@ sub coverageExtract
         {
             $strModulePath .= 'test/src/module/' . substr(${strModuleOutName}, 5);
         }
+        elsif (${strModuleOutName} =~ /^doc\//)
+        {
+            $strModulePath .= "${strModuleOutName}";
+        }
         else
         {
             $strModulePath .= "src/${strModuleOutName}";
@@ -161,7 +169,7 @@ sub coverageExtract
         my $strModuleSourceFile = $strModulePath . '.c' . ($bInc ? '.inc' : '');
 
         executeTest(
-            "${strLCovExe}" . ($bTest ? ' --rc lcov_branch_coverage=0' : '') . " --extract=${strLCovOut} */${strModuleName}.c" .
+            "${strLCovExe}" . ($bTest ? ' --rc lcov_branch_coverage=0' : '') . " --extract=${strLCovOut} *${strModuleName}.c" .
             ($bInc ? '.inc' : '') . " --o=${strLCovOutTmp}");
 
         # Combine with prior run if there was one
@@ -315,6 +323,7 @@ sub coverageValidateAndGenerate
         {
             my $strCoverageFile = $strCodeModule;
             $strCoverageFile =~ s/^test/src/mg;
+            $strCoverageFile =~ s/^doc/doc\/src/mg;
             $strCoverageFile =~ s/^module/test/mg;
             $strCoverageFile = "${strTestResultCoveragePath}/raw/${strCoverageFile}.lcov";
 

test/src/command/test/build.c

@@ -214,6 +214,8 @@ cmdBldPathModule(const String *const moduleName)
     {
         if (strBeginsWithZ(moduleName, "test/"))
             strCatFmt(result, "test/src%s", strZ(strSub(moduleName, 4)));
+        else if (strBeginsWithZ(moduleName, "doc/"))
+            strCatFmt(result, "doc/src%s", strZ(strSub(moduleName, 3)));
         else
             strCatFmt(result, "src/%s", strZ(moduleName));
     }
@@ -534,10 +536,11 @@ testBldUnit(TestBuild *const this)
             "        include_directories(\n"
             "            '.',\n"
             "            '%s/src',\n"
+            "            '%s/doc/src',\n"
             "            '%s/test/src',\n"
             "        ),\n"
             "    dependencies: [\n",
-            strZ(pathRepoRel), strZ(pathRepoRel));
+            strZ(pathRepoRel), strZ(pathRepoRel), strZ(pathRepoRel));
 
         if (testBldBackTrace(this))
         {

test/src/module/doc/buildTest.c

@@ -0,0 +1,331 @@
+/***********************************************************************************************************************************
+Test Documentation Build
+***********************************************************************************************************************************/
+#include "storage/posix/storage.h"
+
+#include "common/harnessStorage.h"
+
+/***********************************************************************************************************************************
+Test Run
+***********************************************************************************************************************************/
+static void
+testRun(void)
+{
+    FUNCTION_HARNESS_VOID();
+
+    // Create default storage object for testing
+    const Storage *const storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
+
+    // *****************************************************************************************************************************
+    if (testBegin("cmdBuild()"))
+    {
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("referenceCommandSection()");
+
+        TEST_RESULT_STR_Z(referenceCommandSection(NULL), "general", "null section remap");
+
+        TEST_RESULT_STR_Z(referenceCommandSection(STRDEF("general")), "general", "general section no remap");
+        TEST_RESULT_STR_Z(referenceCommandSection(STRDEF("log")), "log", "log section no remap");
+        TEST_RESULT_STR_Z(referenceCommandSection(STRDEF("repository")), "repository", "repository section no remap");
+        TEST_RESULT_STR_Z(referenceCommandSection(STRDEF("stanza")), "stanza", "stanza section no remap");
+
+        TEST_RESULT_STR_Z(referenceCommandSection(STRDEF("other")), "command", "other section remap");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("parse and render");
+
+        HRN_STORAGE_PUT_Z(
+            storageTest, "src/build/config/config.yaml",
+            "command:\n"
+            "  backup: {}\n"
+            "\n"
+            "  check: {}\n"
+            "\n"
+            "  restore:\n"
+            "    internal: true\n"
+            "\n"
+            "optionGroup:\n"
+            "  pg: {}\n"
+            "  repo: {}\n"
+            "\n"
+            "option:\n"
+            "  config:\n"
+            "    type: string\n"
+            "    command:\n"
+            "      backup:\n"
+            "        internal: true\n"
+            "      check: {}\n"
+            "\n"
+            "  buffer-size:\n"
+            "    section: global\n"
+            "    beta: true\n"
+            "    type: integer\n"
+            "    default: 1024\n"
+            "    allow-list: [512, 1024, 2048, 4096]\n"
+            "\n"
+            "  internal:\n"
+            "    section: global\n"
+            "    internal: true\n"
+            "    type: integer\n"
+            "    default: 11\n"
+            "\n"
+            "  secure:\n"
+            "    section: global\n"
+            "    type: boolean\n"
+            "    secure: true\n"
+            "    default: false\n"
+            "\n"
+            "  repo-compress-level:\n"
+            "    group: repo\n"
+            "    type: integer\n"
+            "    default: 9\n"
+            "    allow-range: [0, 9]\n"
+            "    command:\n"
+            "      backup: {}\n"
+            "    deprecate:\n"
+            "      repo-compress-level: {}\n"
+            "\n"
+            "  force:\n"
+            "    type: boolean\n"
+            "    default: true\n"
+            "    command:\n"
+            "      backup:\n"
+            "        default: true\n"
+            "    deprecate:\n"
+            "      frc: {}\n"
+            "\n"
+            "  stanza:\n"
+            "    type: string\n"
+            "    default: demo\n"
+            "    required: false\n"
+            "    deprecate:\n"
+            "      stanza: {}\n"
+            "      stanza1: {}\n"
+            "      stanza2: {}\n"
+            "    command:\n"
+            "      backup:\n"
+            "        command-role:\n"
+            "          local: {}\n"
+            "          remote: {}\n"
+            "      check: {}\n"
+            "\n");
+
+        HRN_STORAGE_PUT_Z(
+            storageTest, "src/build/help/help.xml",
+            "<doc>\n"
+            "    <config title=\"Config Title\">\n"
+            "       <description>config description</description>"
+            "\n"
+            "       <text><p>config text</p></text>"
+            "\n"
+            "       <config-section-list>\n"
+            "           <config-section id=\"general\" name=\"General\">\n"
+            "               <text><p>general section</p></text>"
+            "\n"
+            "               <config-key-list>\n"
+            "                   <config-key id=\"buffer-size\" name=\"Buffer Size\">\n"
+            "                       <summary>Buffer size option summary.</summary>\n"
+            "                       <text><p>Buffer size option description.</p></text>\n"
+            "                       <example>128KiB</example>\n"
+            "                       <example>256KiB</example>\n"
+            "                   </config-key>\n"
+            "\n"
+            "                   <config-key id=\"internal\" name=\"Internal\">\n"
+            "                       <summary>Internal option summary.</summary>\n"
+            "                       <text><p>Internal option description</p></text>\n"
+            "                   </config-key>\n"
+            "\n"
+            "                   <config-key id=\"secure\" name=\"Secure\">\n"
+            "                       <summary>Secure option summary.</summary>\n"
+            "                       <text><p>Secure option description</p></text>\n"
+            "                   </config-key>\n"
+            "\n"
+            "                   <config-key id=\"stanza\" section=\"stanza\" name=\"Stanza\">\n"
+            "                       <summary>Stanza option summary.</summary>\n"
+            "                       <text><p>Stanza option description</p></text>\n"
+            "                   </config-key>\n"
+            "               </config-key-list>\n"
+            "           </config-section>\n"
+            "       </config-section-list>\n"
+            "    </config>\n"
+            "\n"
+            "    <operation title=\"Command Title\">\n"
+            "       <description>command description</description>"
+            "       <text><p>command text</p></text>"
+            "\n"
+            "        <operation-general>\n"
+            "           <option-list>\n"
+            "               <option id=\"config\" name=\"Config\">\n"
+            "                   <summary>config option summary.</summary>\n"
+            "                   <text><p>config option description.</p></text>\n"
+            "               </option>\n"
+            "           </option-list>\n"
+            "        </operation-general>\n"
+            "\n"
+            "       <command-list>\n"
+            "           <command id=\"backup\" name=\"Backup\">\n"
+            "               <summary>backup command summary.</summary>\n"
+            "               <text><p>Backup command description.</p></text>\n"
+            "\n"
+            "               <option-list>\n"
+            "                   <option id=\"force\" name=\"Force Backup\">\n"
+            "                       <summary>Force option command backup summary.</summary>\n"
+            "                       <text><p>Force option command backup description.</p></text>\n"
+            "                       <example>n</example>"
+            "                       <example>y</example>"
+            "                   </option>\n"
+            "\n"
+            "                   <option id=\"repo-compress-level\" name=\"Repo Compress Level Backup\">\n"
+            "                       <summary>Repo compress level option command backup summary.</summary>\n"
+            "                       <text><p>Repo compress level option command backup description.</p></text>\n"
+            "                       <example>4</example>"
+            "                   </option>\n"
+            "               </option-list>\n"
+            "           </command>\n"
+            "\n"
+            "           <command id=\"check\" name=\"Check\">\n"
+            "               <summary>Check command summary.</summary>\n"
+            "               <text><p>Check command description.</p></text>\n"
+            "           </command>\n"
+            "\n"
+            "           <command id=\"restore\" name=\"Restore\">\n"
+            "               <summary>Restore command summary.</summary>\n"
+            "               <text><p>Restore command description.</p></text>\n"
+            "           </command>\n"
+            "       </command-list>\n"
+            "    </operation>\n"
+            "</doc>\n");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("build documentation");
+
+        TEST_RESULT_VOID(cmdBuild(TEST_PATH_STR), "write files");
+        TEST_STORAGE_EXISTS(storageTest, "doc/output/xml/configuration.xml");
+        TEST_STORAGE_EXISTS(storageTest, "doc/output/xml/command.xml");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("configuration.xml");
+
+        TEST_STORAGE_GET(
+            storageTest,
+            "doc/output/xml/configuration.xml",
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+            "<!DOCTYPE doc SYSTEM \"doc.dtd\">\n"
+            "<doc title=\"{[project]}\" subtitle=\"Config Title\" toc=\"y\">"
+            // {uncrustify_off - indentation}
+                "<description>config description</description>"
+                "<section id=\"introduction\">"
+                    "<title>Introduction</title>"
+                    "<text><p>config text</p></text>"
+                "</section>"
+                "<section id=\"section-general\">"
+                    "<title>General Options (<id>general</id>)</title>"
+                    "<text><p>general section</p></text>"
+                    "<section id=\"option-buffer-size\">"
+                        "<title>Buffer Size Option (<id>--buffer-size</id>)</title>"
+                        "<p>Buffer size option summary.</p>"
+                        "<p>FOR BETA TESTING ONLY. DO NOT USE IN PRODUCTION.</p>"
+                        "<p><p>Buffer size option description.</p></p>"
+                        "<code-block>default: 1024\n"
+                        "example: buffer-size=128KiB\n"
+                        "example: buffer-size=256KiB</code-block>"
+                    "</section>"
+                    "<section id=\"option-config\">"
+                        "<title>Config Option (<id>--config</id>)</title>"
+                        "<p>config option summary.</p>"
+                        "<p><p>config option description.</p></p>"
+                    "</section>"
+                    "<section id=\"option-secure\">"
+                        "<title>Secure Option (<id>--secure</id>)</title>"
+                        "<p>Secure option summary.</p>"
+                        "<p><p>Secure option description</p></p>"
+                        "<code-block>default: n</code-block>"
+                    "</section>"
+                "</section>"
+            // {uncrustify_on}
+            "</doc>\n");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("command.xml");
+
+        TEST_STORAGE_GET(
+            storageTest,
+            "doc/output/xml/command.xml",
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+            "<!DOCTYPE doc SYSTEM \"doc.dtd\">\n"
+            "<doc title=\"{[project]}\" subtitle=\"Command Title\" toc=\"y\">"
+            // {uncrustify_off - indentation}
+                "<description>command description</description>"
+                "<section id=\"introduction\"><title>Introduction</title>"
+                    "<text><p>command text</p></text>"
+                "</section>"
+                "<section id=\"command-backup\">"
+                    "<title>Backup Command (<id>backup</id>)</title>"
+                    "<text><p>Backup command description.</p></text>"
+                    "<section id=\"category-command\" toc=\"n\">"
+                        "<title>Command Options</title>"
+                        "<section id=\"option-force\">"
+                            "<title>Force Backup Option (<id>--force</id>)</title>"
+                            "<p>Force option command backup summary.</p>"
+                            "<p><p>Force option command backup description.</p></p>"
+                            "<code-block>default: y\n"
+                            "example: --no-force --force</code-block>"
+                            "<p>Deprecated Name: frc</p>"
+                        "</section>"
+                        "<section id=\"option-repo-compress-level\">"
+                            "<title>Repo Compress Level Backup Option (<id>--repo-compress-level</id>)</title>"
+                            "<p>Repo compress level option command backup summary.</p>"
+                            "<p><p>Repo compress level option command backup description.</p></p>"
+                            "<code-block>default: 9\n"
+                            "allowed: 0-9\n"
+                            "example: --repo1-compress-level=4</code-block>"
+                        "</section>"
+                    "</section>"
+                    "<section id=\"category-general\" toc=\"n\">"
+                        "<title>General Options</title>"
+                        "<section id=\"option-buffer-size\">"
+                            "<title>Buffer Size Option (<id>--buffer-size</id>)</title>"
+                            "<p>Buffer size option summary.</p>"
+                            "<p>FOR BETA TESTING ONLY. DO NOT USE IN PRODUCTION.</p>"
+                            "<p><p>Buffer size option description.</p></p>"
+                            "<code-block>default: 1024\n"
+                            "example: --buffer-size=128KiB --buffer-size=256KiB</code-block>"
+                        "</section>"
+                    "</section>"
+                "</section>"
+                "<section id=\"command-check\">"
+                    "<title>Check Command (<id>check</id>)</title>"
+                    "<text><p>Check command description.</p></text>"
+                    "<section id=\"category-general\" toc=\"n\">"
+                        "<title>General Options</title>"
+                        "<section id=\"option-buffer-size\">"
+                            "<title>Buffer Size Option (<id>--buffer-size</id>)</title>"
+                            "<p>Buffer size option summary.</p>"
+                            "<p>FOR BETA TESTING ONLY. DO NOT USE IN PRODUCTION.</p>"
+                            "<p><p>Buffer size option description.</p></p>"
+                            "<code-block>default: 1024\n"
+                            "example: --buffer-size=128KiB --buffer-size=256KiB</code-block>"
+                        "</section>"
+                        "<section id=\"option-config\">"
+                            "<title>Config Option (<id>--config</id>)</title>"
+                            "<p>config option summary.</p>"
+                            "<p><p>config option description.</p></p>"
+                        "</section>"
+                    "</section>"
+                    "<section id=\"category-stanza\" toc=\"n\">"
+                        "<title>Stanza Options</title>"
+                        "<section id=\"option-stanza\">"
+                            "<title>Stanza Option (<id>--stanza</id>)</title>"
+                            "<p>Stanza option summary.</p>"
+                            "<p><p>Stanza option description</p></p>"
+                            "<code-block>default: demo</code-block>"
+                            "<p>Deprecated Names: stanza1, stanza2</p>"
+                        "</section>"
+                    "</section>"
+                "</section>"
+            // {uncrustify_on}
+            "</doc>\n");
+    }
+
+    FUNCTION_HARNESS_RETURN_VOID();
+}

test/src/module/test/testTest.c

@@ -206,6 +206,7 @@ testRun(void)
             "        include:\n"
             "          - common/error/error\n"
             "          - test/common/include\n"
+            "          - doc/command/build/build\n"
             "\n"
             "integration:\n"
             "  - name: mock\n"
@@ -258,6 +259,8 @@ testRun(void)
             "test/src/module/common/stackTraceTest.c",
             "test/src/test.c");
 
+        HRN_STORAGE_PUT_EMPTY(storageTest, "repo/doc/src/command/build/build.c");
+
         TEST_RESULT_VOID(
             cmdTest(
                 STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3,
@@ -307,6 +310,7 @@ testRun(void)
                         "        include_directories(\n"
                         "            '.',\n"
                         "            '../../../repo/src',\n"
+                        "            '../../../repo/doc/src',\n"
                         "            '../../../repo/test/src',\n"
                         "        ),\n"
                         "    dependencies: [\n"
@@ -423,6 +427,7 @@ testRun(void)
                         "        include_directories(\n"
                         "            '.',\n"
                         "            '../../../repo/src',\n"
+                        "            '../../../repo/doc/src',\n"
                         "            '../../../repo/test/src',\n"
                         "        ),\n"
                         "    dependencies: [\n"
@@ -634,6 +639,7 @@ testRun(void)
                         "        include_directories(\n"
                         "            '.',\n"
                         "            '../../../repo/src',\n"
+                        "            '../../../repo/doc/src',\n"
                         "            '../../../repo/test/src',\n"
                         "        ),\n"
                         "    dependencies: [\n"
@@ -688,7 +694,8 @@ testRun(void)
                     STRDEF(
                         "#include \"test/src/common/harnessShim.c\"\n"
                         "#include \"test/src/common/harnessError.c\"\n"
-                        "#include \"../../../repo/test/src/common/include.c\""));
+                        "#include \"../../../repo/test/src/common/include.c\"\n"
+                        "#include \"../../../repo/doc/src/command/build/build.c\""));
                 strReplace(
                     testCDup, STRDEF("{[C_TEST_INCLUDE]}"), STRDEF("#include \"../../../repo/test/src/module/test/shimTest.c\""));
                 strReplace(
@@ -806,6 +813,7 @@ testRun(void)
                         "        include_directories(\n"
                         "            '.',\n"
                         "            '../../../repo/src',\n"
+                        "            '../../../repo/doc/src',\n"
                         "            '../../../repo/test/src',\n"
                         "        ),\n"
                         "    dependencies: [\n"