From 15d04ca19c0bd1aafe9d4389e8429bcc392e842d Mon Sep 17 00:00:00 2001
From: David Steele <david@pgmasters.net>
Date: Thu, 12 Sep 2019 16:29:50 -0400
Subject: [PATCH] Add recursion and json output to the ls command.

These features finally make the ls command practical.

Currently the JSON contains only name, type, and size.  We may add more fields in the future, but these seem like the minimum needed to be useful.
---
 build/lib/pgBackRestBuild/Config/Data.pm |  37 ++++--
 doc/xml/reference.xml                    |  25 +++-
 lib/pgBackRest/LibCAuto.pm               |   8 ++
 src/Makefile.in                          |   2 +-
 src/command/storage/list.c               | 152 ++++++++++++++++++++---
 src/config/config.auto.c                 |   9 ++
 src/config/config.auto.h                 |   5 +-
 src/config/define.auto.c                 |  63 +++++++++-
 src/config/define.auto.h                 |   1 +
 src/config/parse.auto.c                  |   8 ++
 src/perl/embed.auto.c                    |   8 ++
 test/src/module/command/storageTest.c    | 126 ++++++++++++++++---
 12 files changed, 397 insertions(+), 47 deletions(-)

diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm
index b17e1daf4..68cd9cebb 100644
--- a/build/lib/pgBackRestBuild/Config/Data.pm
+++ b/build/lib/pgBackRestBuild/Config/Data.pm
@@ -163,6 +163,8 @@ use constant CFGOPT_HOST_ID                                         => 'host-id'
 #-----------------------------------------------------------------------------------------------------------------------------------
 use constant CFGOPT_FILTER                                          => 'filter';
     push @EXPORT, qw(CFGOPT_FILTER);
+use constant CFGOPT_RECURSE                                         => 'recurse';
+    push @EXPORT, qw(CFGOPT_RECURSE);
 use constant CFGOPT_SORT                                            => 'sort';
     push @EXPORT, qw(CFGOPT_SORT);
 
@@ -415,10 +417,10 @@ use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC                 => 'aes-256-
 
 # Info output
 #-----------------------------------------------------------------------------------------------------------------------------------
-use constant CFGOPTVAL_INFO_OUTPUT_TEXT                             => 'text';
-    push @EXPORT, qw(CFGOPTVAL_INFO_OUTPUT_TEXT);
-use constant CFGOPTVAL_INFO_OUTPUT_JSON                             => 'json';
-    push @EXPORT, qw(CFGOPTVAL_INFO_OUTPUT_JSON);
+use constant CFGOPTVAL_OUTPUT_TEXT                                  => 'text';
+    push @EXPORT, qw(CFGOPTVAL_OUTPUT_TEXT);
+use constant CFGOPTVAL_OUTPUT_JSON                                  => 'json';
+    push @EXPORT, qw(CFGOPTVAL_OUTPUT_JSON);
 
 # Restore type
 #-----------------------------------------------------------------------------------------------------------------------------------
@@ -1013,11 +1015,21 @@ my %hConfigDefine =
         {
             &CFGCMD_INFO =>
             {
-                &CFGDEF_DEFAULT => CFGOPTVAL_INFO_OUTPUT_TEXT,
+                &CFGDEF_DEFAULT => CFGOPTVAL_OUTPUT_TEXT,
                 &CFGDEF_ALLOW_LIST =>
                 [
-                    &CFGOPTVAL_INFO_OUTPUT_TEXT,
-                    &CFGOPTVAL_INFO_OUTPUT_JSON,
+                    &CFGOPTVAL_OUTPUT_TEXT,
+                    &CFGOPTVAL_OUTPUT_JSON,
+                ]
+            },
+
+            &CFGCMD_STORAGE_LIST =>
+            {
+                &CFGDEF_DEFAULT => CFGOPTVAL_OUTPUT_TEXT,
+                &CFGDEF_ALLOW_LIST =>
+                [
+                    &CFGOPTVAL_OUTPUT_TEXT,
+                    &CFGOPTVAL_OUTPUT_JSON,
                 ]
             }
         }
@@ -1085,12 +1097,23 @@ my %hConfigDefine =
         }
     },
 
+    &CFGOPT_RECURSE =>
+    {
+        &CFGDEF_TYPE => CFGDEF_TYPE_BOOLEAN,
+        &CFGDEF_DEFAULT => false,
+        &CFGDEF_COMMAND =>
+        {
+            &CFGCMD_STORAGE_LIST => {},
+        }
+    },
+
     &CFGOPT_SORT =>
     {
         &CFGDEF_TYPE => CFGDEF_TYPE_STRING,
         &CFGDEF_DEFAULT => 'asc',
         &CFGDEF_ALLOW_LIST =>
         [
+            'none',
             'asc',
             'desc',
         ],
diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml
index 3e089f039..65babd1db 100644
--- a/doc/xml/reference.xml
+++ b/doc/xml/reference.xml
@@ -1243,14 +1243,37 @@
                         <example>(F|D|I)$</example>
                     </option>
 
+                    <!-- OPERATION - LS COMMAND - OUTPUT OPTION -->
+                    <option id="output" name="Output">
+                        <summary>Output format.</summary>
+
+                        <text>The following output types are supported:
+                        <ul>
+                            <li><id>text</id> - Simple list with one file/link/path name on each line.</li>
+                            <li><id>json</id> - Detailed file/link/path information in JSON format.</li>
+                        </ul></text>
+
+                        <example>json</example>
+                    </option>
+
+                    <!-- OPERATION - LS COMMAND - RECURSE OPTION -->
+                    <option id="recurse" name="Recurse Subpaths">
+                        <summary>Include all subpaths in output.</summary>
+
+                        <text>All subpaths and their files will be included in the output.</text>
+
+                        <example>y</example>
+                    </option>
+
                     <!-- OPERATION - LS COMMAND - SORT OPTION -->
                     <option id="sort" name="Sort Output">
-                        <summary>Sort output ascending/descending.</summary>
+                        <summary>Sort output ascending, descending, or none.</summary>
 
                         <text>The following sort types are supported:
                         <ul>
                             <li><id>asc</id> - sort ascending.</li>
                             <li><id>desc</id> - sort descending.</li>
+                            <li><id>none</id> - no sorting.</li>
                         </ul></text>
 
                         <example>desc</example>
diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm
index 18813f586..d31fe4a8e 100644
--- a/lib/pgBackRest/LibCAuto.pm
+++ b/lib/pgBackRest/LibCAuto.pm
@@ -14,6 +14,9 @@ sub libcAutoConstant
         CFGOPTVAL_INFO_OUTPUT_TEXT                                       => 'text',
         CFGOPTVAL_INFO_OUTPUT_JSON                                       => 'json',
 
+        CFGOPTVAL_LS_OUTPUT_TEXT                                         => 'text',
+        CFGOPTVAL_LS_OUTPUT_JSON                                         => 'json',
+
         CFGOPTVAL_REPO_CIPHER_TYPE_NONE                                  => 'none',
         CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC                           => 'aes-256-cbc',
 
@@ -25,6 +28,7 @@ sub libcAutoConstant
         CFGOPTVAL_REPO_TYPE_POSIX                                        => 'posix',
         CFGOPTVAL_REPO_TYPE_S3                                           => 's3',
 
+        CFGOPTVAL_SORT_NONE                                              => 'none',
         CFGOPTVAL_SORT_ASC                                               => 'asc',
         CFGOPTVAL_SORT_DESC                                              => 'desc',
 
@@ -80,6 +84,8 @@ sub libcAutoExportTag
         [
             'CFGOPTVAL_INFO_OUTPUT_TEXT',
             'CFGOPTVAL_INFO_OUTPUT_JSON',
+            'CFGOPTVAL_LS_OUTPUT_TEXT',
+            'CFGOPTVAL_LS_OUTPUT_JSON',
             'CFGOPTVAL_REPO_CIPHER_TYPE_NONE',
             'CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC',
             'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL',
@@ -88,6 +94,7 @@ sub libcAutoExportTag
             'CFGOPTVAL_REPO_TYPE_CIFS',
             'CFGOPTVAL_REPO_TYPE_POSIX',
             'CFGOPTVAL_REPO_TYPE_S3',
+            'CFGOPTVAL_SORT_NONE',
             'CFGOPTVAL_SORT_ASC',
             'CFGOPTVAL_SORT_DESC',
             'CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE',
@@ -249,6 +256,7 @@ sub libcAutoExportTag
             'CFGOPT_PROCESS_MAX',
             'CFGOPT_PROTOCOL_TIMEOUT',
             'CFGOPT_RECOVERY_OPTION',
+            'CFGOPT_RECURSE',
             'CFGOPT_REPO_CIPHER_PASS',
             'CFGOPT_REPO_CIPHER_TYPE',
             'CFGOPT_REPO_HARDLINK',
diff --git a/src/Makefile.in b/src/Makefile.in
index 554a6203e..59142ce27 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -294,7 +294,7 @@ command/stanza/delete.o: command/stanza/delete.c build.auto.h command/backup/com
 command/stanza/upgrade.o: command/stanza/upgrade.c build.auto.h command/control/common.h command/stanza/common.h command/stanza/upgrade.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/upgrade.c -o command/stanza/upgrade.o
 
-command/storage/list.o: command/storage/list.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
+command/storage/list.o: command/storage/list.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/storage/list.c -o command/storage/list.o
 
 common/compress/gzip/common.o: common/compress/gzip/common.c build.auto.h common/assert.h common/compress/gzip/common.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/convert.h
diff --git a/src/command/storage/list.c b/src/command/storage/list.c
index c5f8a433a..248746173 100644
--- a/src/command/storage/list.c
+++ b/src/command/storage/list.c
@@ -10,40 +10,155 @@ Storage List Command
 #include "common/memContext.h"
 #include "common/log.h"
 #include "common/memContext.h"
+#include "common/type/json.h"
 #include "common/type/string.h"
 #include "config/config.h"
 #include "storage/helper.h"
 
 /***********************************************************************************************************************************
-Render storage list as a string
+Render storage list
 ***********************************************************************************************************************************/
-static String *
-storageListRender(void)
+typedef struct StorageListRenderCallbackData
 {
-    FUNCTION_LOG_VOID(logLevelDebug);
+    IoWrite *write;                                                 // Where to write output
+    bool json;                                                      // Is this json output?
+    bool first;                                                     // Is this the first item?
+} StorageListRenderCallbackData;
 
-    // Get the path if passed
+void
+storageListRenderCallback(void *data, const StorageInfo *info)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM_P(VOID, data);
+        FUNCTION_TEST_PARAM(STORAGE_INFO, info);
+    FUNCTION_TEST_END();
+
+    ASSERT(data != NULL);
+    ASSERT(info != NULL);
+
+    StorageListRenderCallbackData *listData = (StorageListRenderCallbackData *)data;
+
+    // Skip . path when output is text
+    if (info->type == storageTypePath && strEq(info->name, DOT_STR) && !listData->json)
+    {
+        FUNCTION_TEST_RETURN_VOID();
+        return;
+    }
+
+    // Add seperator character
+    if (!listData->first)
+    {
+        if (listData->json)
+            ioWrite(listData->write, COMMA_BUF);
+        else
+            ioWrite(listData->write, LF_BUF);
+    }
+    else
+        listData->first = false;
+
+    // Render in json
+    if (listData->json)
+    {
+        ioWriteStr(listData->write, jsonFromStr(info->name));
+        ioWrite(listData->write, BUFSTRDEF(":{\"type\":\""));
+
+        switch (info->type)
+        {
+            case storageTypeFile:
+            {
+                ioWrite(listData->write, BUFSTRDEF("file\""));
+                break;
+            }
+
+            case storageTypeLink:
+            {
+                ioWrite(listData->write, BUFSTRDEF("link\""));
+                break;
+            }
+
+            case storageTypePath:
+            {
+                ioWrite(listData->write, BUFSTRDEF("path\""));
+                break;
+            }
+
+            case storageTypeSpecial:
+            {
+                ioWrite(listData->write, BUFSTRDEF("special\""));
+                break;
+            }
+        }
+
+        if (info->type == storageTypeFile)
+            ioWriteStr(listData->write, strNewFmt(",\"size\":%" PRIu64, info->size));
+
+        if (info->type == storageTypeLink)
+            ioWriteStr(listData->write, strNewFmt(",\"destination\":%s", strPtr(jsonFromStr(info->linkDestination))));
+
+        ioWrite(listData->write, BRACER_BUF);
+    }
+    // Render in text
+    else
+    {
+        ioWrite(listData->write, BUFSTR(info->name));
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
+static void
+storageListRender(IoWrite *write)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug)
+        FUNCTION_LOG_PARAM(IO_WRITE, write);
+
+    FUNCTION_LOG_END();
+
+    // Get sort order
+    SortOrder sortOrder = sortOrderAsc;
+
+    if (strEqZ(cfgOptionStr(cfgOptSort), "desc"))
+        sortOrder = sortOrderDesc;
+    else if (!strEqZ(cfgOptionStr(cfgOptSort), "asc"))
+    {
+        ASSERT(strEqZ(cfgOptionStr(cfgOptSort), "none"));
+        sortOrder = sortOrderNone;
+    }
+
+    // Get path
     const String *path = NULL;
 
     if (strLstSize(cfgCommandParam()) == 1)
         path = strLstGet(cfgCommandParam(), 0);
+    else if (strLstSize(cfgCommandParam()) > 1)
+        THROW(ParamInvalidError, "only one path may be specified");
 
-    // Output the list
-    String *result = strNew("");
-    StringList *list = storageListP(storageRepo(), path, .nullOnMissing = true, .expression = cfgOptionStr(cfgOptFilter));
+    // Get output
+    bool json = strEqZ(cfgOptionStr(cfgOptOutput), "json") ? true : false;
 
-    if (list != NULL)
+    // Render the info list
+    StorageListRenderCallbackData data =
     {
-        strLstSort(list, strEqZ(cfgOptionStr(cfgOptSort), "asc") ? sortOrderAsc : sortOrderDesc);
+        .write = write,
+        .json = json,
+        .first = true,
+    };
 
-        for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
-        {
-            strCat(result, strPtr(strLstGet(list, listIdx)));
-            strCat(result, "\n");
-        }
-    }
+    ioWriteOpen(data.write);
 
-    FUNCTION_LOG_RETURN(STRING, result);
+    if (data.json)
+        ioWrite(data.write, BRACEL_BUF);
+
+    storageInfoListP(
+        storageRepo(), path, storageListRenderCallback, &data, .sortOrder = sortOrder, .expression = cfgOptionStr(cfgOptFilter),
+        .recurse = cfgOptionBool(cfgOptRecurse));
+
+    if (data.json)
+        ioWrite(data.write, BRACER_BUF);
+
+    ioWriteClose(data.write);
+
+    FUNCTION_LOG_RETURN_VOID();
 }
 
 /***********************************************************************************************************************************
@@ -56,7 +171,8 @@ cmdStorageList(void)
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ioHandleWriteOneStr(STDOUT_FILENO, storageListRender());
+        storageListRender(ioHandleWriteNew(STRDEF("stdout"), STDOUT_FILENO));
+        ioHandleWriteOneStr(STDOUT_FILENO, LF_STR);
     }
     MEM_CONTEXT_TEMP_END();
 
diff --git a/src/config/config.auto.c b/src/config/config.auto.c
index 490f58a03..dc20d8827 100644
--- a/src/config/config.auto.c
+++ b/src/config/config.auto.c
@@ -425,6 +425,7 @@ STRING_EXTERN(CFGOPT_PROCESS_STR,                                   CFGOPT_PROCE
 STRING_EXTERN(CFGOPT_PROCESS_MAX_STR,                               CFGOPT_PROCESS_MAX);
 STRING_EXTERN(CFGOPT_PROTOCOL_TIMEOUT_STR,                          CFGOPT_PROTOCOL_TIMEOUT);
 STRING_EXTERN(CFGOPT_RECOVERY_OPTION_STR,                           CFGOPT_RECOVERY_OPTION);
+STRING_EXTERN(CFGOPT_RECURSE_STR,                                   CFGOPT_RECURSE);
 STRING_EXTERN(CFGOPT_REPO1_CIPHER_PASS_STR,                         CFGOPT_REPO1_CIPHER_PASS);
 STRING_EXTERN(CFGOPT_REPO1_CIPHER_TYPE_STR,                         CFGOPT_REPO1_CIPHER_TYPE);
 STRING_EXTERN(CFGOPT_REPO1_HARDLINK_STR,                            CFGOPT_REPO1_HARDLINK);
@@ -1459,6 +1460,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
         CONFIG_OPTION_DEFINE_ID(cfgDefOptRecoveryOption)
     )
 
+    //------------------------------------------------------------------------------------------------------------------------------
+    CONFIG_OPTION
+    (
+        CONFIG_OPTION_NAME(CFGOPT_RECURSE)
+        CONFIG_OPTION_INDEX(0)
+        CONFIG_OPTION_DEFINE_ID(cfgDefOptRecurse)
+    )
+
     //------------------------------------------------------------------------------------------------------------------------------
     CONFIG_OPTION
     (
diff --git a/src/config/config.auto.h b/src/config/config.auto.h
index e975d1a32..f23eedb58 100644
--- a/src/config/config.auto.h
+++ b/src/config/config.auto.h
@@ -299,6 +299,8 @@ Option constants
     STRING_DECLARE(CFGOPT_PROTOCOL_TIMEOUT_STR);
 #define CFGOPT_RECOVERY_OPTION                                      "recovery-option"
     STRING_DECLARE(CFGOPT_RECOVERY_OPTION_STR);
+#define CFGOPT_RECURSE                                              "recurse"
+    STRING_DECLARE(CFGOPT_RECURSE_STR);
 #define CFGOPT_REPO1_CIPHER_PASS                                    "repo1-cipher-pass"
     STRING_DECLARE(CFGOPT_REPO1_CIPHER_PASS_STR);
 #define CFGOPT_REPO1_CIPHER_TYPE                                    "repo1-cipher-type"
@@ -388,7 +390,7 @@ Option constants
 #define CFGOPT_TYPE                                                 "type"
     STRING_DECLARE(CFGOPT_TYPE_STR);
 
-#define CFG_OPTION_TOTAL                                            167
+#define CFG_OPTION_TOTAL                                            168
 
 /***********************************************************************************************************************************
 Command enum
@@ -545,6 +547,7 @@ typedef enum
     cfgOptProcessMax,
     cfgOptProtocolTimeout,
     cfgOptRecoveryOption,
+    cfgOptRecurse,
     cfgOptRepoCipherPass,
     cfgOptRepoCipherType,
     cfgOptRepoHardlink,
diff --git a/src/config/define.auto.c b/src/config/define.auto.c
index 3c9754c7b..58a44aa89 100644
--- a/src/config/define.auto.c
+++ b/src/config/define.auto.c
@@ -1901,6 +1901,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
         CFGDEFDATA_OPTION_COMMAND_LIST
         (
             CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLs)
         )
 
         CFGDEFDATA_OPTION_OPTIONAL_LIST
@@ -1926,6 +1927,28 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
                     "* json - Exhaustive machine-readable backup information in JSON format."
                 )
             )
+
+            CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRIDE
+            (
+                CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdLs)
+
+                CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
+                (
+                    "text",
+                    "json"
+                )
+
+                CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("text")
+
+                CFGDEFDATA_OPTION_OPTIONAL_HELP_SUMMARY("Output format.")
+                CFGDEFDATA_OPTION_OPTIONAL_HELP_DESCRIPTION
+                (
+                    "The following output types are supported:\n"
+                    "\n"
+                    "* text - Simple list with one file/link/path name on each line.\n"
+                    "* json - Detailed file/link/path information in JSON format."
+                )
+            )
         )
     )
 
@@ -2626,6 +2649,40 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
         )
     )
 
+    // -----------------------------------------------------------------------------------------------------------------------------
+    CFGDEFDATA_OPTION
+    (
+        CFGDEFDATA_OPTION_NAME("recurse")
+        CFGDEFDATA_OPTION_REQUIRED(true)
+        CFGDEFDATA_OPTION_SECTION(cfgDefSectionCommandLine)
+        CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeBoolean)
+        CFGDEFDATA_OPTION_INTERNAL(false)
+
+        CFGDEFDATA_OPTION_INDEX_TOTAL(1)
+        CFGDEFDATA_OPTION_SECURE(false)
+
+        CFGDEFDATA_OPTION_COMMAND_LIST
+        (
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLs)
+        )
+
+        CFGDEFDATA_OPTION_OPTIONAL_LIST
+        (
+            CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("0")
+
+            CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRIDE
+            (
+                CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdLs)
+
+                CFGDEFDATA_OPTION_OPTIONAL_HELP_SUMMARY("Include all subpaths in output.")
+                CFGDEFDATA_OPTION_OPTIONAL_HELP_DESCRIPTION
+                (
+                    "All subpaths and their files will be included in the output."
+                )
+            )
+        )
+    )
+
     // -----------------------------------------------------------------------------------------------------------------------------
     CFGDEFDATA_OPTION
     (
@@ -4079,6 +4136,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
         (
             CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
             (
+                "none",
                 "asc",
                 "desc"
             )
@@ -4089,13 +4147,14 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
             (
                 CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdLs)
 
-                CFGDEFDATA_OPTION_OPTIONAL_HELP_SUMMARY("Sort output ascending/descending.")
+                CFGDEFDATA_OPTION_OPTIONAL_HELP_SUMMARY("Sort output ascending, descending, or none.")
                 CFGDEFDATA_OPTION_OPTIONAL_HELP_DESCRIPTION
                 (
                     "The following sort types are supported:\n"
                     "\n"
                     "* asc - sort ascending.\n"
-                    "* desc - sort descending."
+                    "* desc - sort descending.\n"
+                    "* none - no sorting."
                 )
             )
         )
diff --git a/src/config/define.auto.h b/src/config/define.auto.h
index 24ede6967..168fcd414 100644
--- a/src/config/define.auto.h
+++ b/src/config/define.auto.h
@@ -105,6 +105,7 @@ typedef enum
     cfgDefOptProcessMax,
     cfgDefOptProtocolTimeout,
     cfgDefOptRecoveryOption,
+    cfgDefOptRecurse,
     cfgDefOptRepoCipherPass,
     cfgDefOptRepoCipherType,
     cfgDefOptRepoHardlink,
diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c
index 185c69c93..546c2bac1 100644
--- a/src/config/parse.auto.c
+++ b/src/config/parse.auto.c
@@ -1638,6 +1638,13 @@ static const struct option optionList[] =
         .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRecoveryOption,
     },
 
+    // recurse option
+    // -----------------------------------------------------------------------------------------------------------------------------
+    {
+        .name = CFGOPT_RECURSE,
+        .val = PARSE_OPTION_FLAG | cfgOptRecurse,
+    },
+
     // repo-cipher-pass option and deprecations
     // -----------------------------------------------------------------------------------------------------------------------------
     {
@@ -2394,6 +2401,7 @@ static const ConfigOption optionResolveOrder[] =
     cfgOptProcess,
     cfgOptProcessMax,
     cfgOptProtocolTimeout,
+    cfgOptRecurse,
     cfgOptRepoCipherType,
     cfgOptRepoHardlink,
     cfgOptRepoHost,
diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c
index 2d2e41ad8..d951a4488 100644
--- a/src/perl/embed.auto.c
+++ b/src/perl/embed.auto.c
@@ -7822,6 +7822,9 @@ static const EmbeddedModule embeddedModule[] =
             "CFGOPTVAL_INFO_OUTPUT_TEXT                                       => 'text',\n"
             "CFGOPTVAL_INFO_OUTPUT_JSON                                       => 'json',\n"
             "\n"
+            "CFGOPTVAL_LS_OUTPUT_TEXT                                         => 'text',\n"
+            "CFGOPTVAL_LS_OUTPUT_JSON                                         => 'json',\n"
+            "\n"
             "CFGOPTVAL_REPO_CIPHER_TYPE_NONE                                  => 'none',\n"
             "CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC                           => 'aes-256-cbc',\n"
             "\n"
@@ -7833,6 +7836,7 @@ static const EmbeddedModule embeddedModule[] =
             "CFGOPTVAL_REPO_TYPE_POSIX                                        => 'posix',\n"
             "CFGOPTVAL_REPO_TYPE_S3                                           => 's3',\n"
             "\n"
+            "CFGOPTVAL_SORT_NONE                                              => 'none',\n"
             "CFGOPTVAL_SORT_ASC                                               => 'asc',\n"
             "CFGOPTVAL_SORT_DESC                                              => 'desc',\n"
             "\n"
@@ -7887,6 +7891,8 @@ static const EmbeddedModule embeddedModule[] =
             "[\n"
             "'CFGOPTVAL_INFO_OUTPUT_TEXT',\n"
             "'CFGOPTVAL_INFO_OUTPUT_JSON',\n"
+            "'CFGOPTVAL_LS_OUTPUT_TEXT',\n"
+            "'CFGOPTVAL_LS_OUTPUT_JSON',\n"
             "'CFGOPTVAL_REPO_CIPHER_TYPE_NONE',\n"
             "'CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC',\n"
             "'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL',\n"
@@ -7895,6 +7901,7 @@ static const EmbeddedModule embeddedModule[] =
             "'CFGOPTVAL_REPO_TYPE_CIFS',\n"
             "'CFGOPTVAL_REPO_TYPE_POSIX',\n"
             "'CFGOPTVAL_REPO_TYPE_S3',\n"
+            "'CFGOPTVAL_SORT_NONE',\n"
             "'CFGOPTVAL_SORT_ASC',\n"
             "'CFGOPTVAL_SORT_DESC',\n"
             "'CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE',\n"
@@ -8056,6 +8063,7 @@ static const EmbeddedModule embeddedModule[] =
             "'CFGOPT_PROCESS_MAX',\n"
             "'CFGOPT_PROTOCOL_TIMEOUT',\n"
             "'CFGOPT_RECOVERY_OPTION',\n"
+            "'CFGOPT_RECURSE',\n"
             "'CFGOPT_REPO_CIPHER_PASS',\n"
             "'CFGOPT_REPO_CIPHER_TYPE',\n"
             "'CFGOPT_REPO_HARDLINK',\n"
diff --git a/test/src/module/command/storageTest.c b/test/src/module/command/storageTest.c
index 3404d120b..f8d26660f 100644
--- a/test/src/module/command/storageTest.c
+++ b/test/src/module/command/storageTest.c
@@ -1,6 +1,7 @@
 /***********************************************************************************************************************************
 Test Storage Commands
 ***********************************************************************************************************************************/
+#include "common/io/bufferWrite.h"
 #include "storage/posix/storage.h"
 
 #include "common/harnessConfig.h"
@@ -23,33 +24,116 @@ testRun(void)
         StringList *argList = strLstNew();
         strLstAddZ(argList, "pgbackrest");
         strLstAdd(argList, strNewFmt("--repo-path=%s/repo", testPath()));
+        strLstAddZ(argList, "--output=text");
+        strLstAddZ(argList, "--sort=none");
         strLstAddZ(argList, "ls");
         harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
 
-        TEST_RESULT_STR(strPtr(storageListRender()), "", "missing directory");
+        // Missing directory
+        // -------------------------------------------------------------------------------------------------------------------------
+        Buffer *output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "missing directory (text)");
+        TEST_RESULT_STR(strNewBuf(output), "", "    check output");
 
-        storagePathCreateNP(storageTest, strNew("repo"));
-        TEST_RESULT_STR(strPtr(storageListRender()), "", "empty directory");
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "missing directory (json)");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "{}", "    check output");
+
+        // Empty directory
+        // -------------------------------------------------------------------------------------------------------------------------
+        storagePathCreateP(storageTest, strNew("repo"), .mode = 0700);
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "empty directory (text)");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "", "    check output");
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "empty directory (json)");
+        TEST_RESULT_STR_Z(
+            strNewBuf(output),
+                "{"
+                    "\".\":{\"type\":\"path\"}"
+                "}",
+            "    check output");
+
+        // Add path and file
+        // -------------------------------------------------------------------------------------------------------------------------
+        cfgOptionSet(cfgOptSort, cfgSourceParam, VARSTRDEF("asc"));
 
         storagePathCreateNP(storageTest, strNew("repo/bbb"));
-        storagePutNP(storageNewWriteNP(storageTest, strNew("repo/aaa")), NULL);
-        TEST_RESULT_STR(strPtr(storageListRender()), "aaa\nbbb\n", "list files");
+        ASSERT(system(strPtr(strNewFmt("sudo chown :77777 %s/repo/bbb", testPath()))) == 0);
 
+        storagePutNP(storageNewWriteNP(storageTest, strNew("repo/aaa")), BUFSTRDEF("TESTDATA"));
+        ASSERT(system(strPtr(strNewFmt("sudo chown 77777 %s/repo/aaa", testPath()))) == 0);
+        ASSERT(system(strPtr(strNewFmt("sudo chmod 000 %s/repo/aaa", testPath()))) == 0);
+
+        storagePutNP(storageNewWriteNP(storageTest, strNew("repo/bbb/ccc")), BUFSTRDEF("TESTDATA2"));
+
+        ASSERT(system(strPtr(strNewFmt("ln -s ../bbb %s/repo/link", testPath()))) == 0);
+        ASSERT(system(strPtr(strNewFmt("mkfifo %s/repo/pipe", testPath()))) == 0);
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (text)");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "aaa\nbbb\nlink\npipe", "    check output");
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (text)");
+        TEST_RESULT_STR_Z(
+            strNewBuf(output),
+                "{"
+                    "\".\":{\"type\":\"path\"},"
+                    "\"aaa\":{\"type\":\"file\",\"size\":8},"
+                    "\"bbb\":{\"type\":\"path\"},"
+                    "\"link\":{\"type\":\"link\",\"destination\":\"../bbb\"},"
+                    "\"pipe\":{\"type\":\"special\"}"
+                "}",
+            "    check output");
+
+        // Reverse sort
+        // -------------------------------------------------------------------------------------------------------------------------
+        cfgOptionSet(cfgOptSort, cfgSourceParam, VARSTRDEF("desc"));
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (text)");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "pipe\nlink\nbbb\naaa", "    check output");
+
+        // Recurse
+        // -------------------------------------------------------------------------------------------------------------------------
+        cfgOptionValidSet(cfgOptRecurse, true);
+        cfgOptionSet(cfgOptRecurse, cfgSourceParam, VARBOOL(true));
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "filter");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "pipe\nlink\nbbb/ccc\nbbb\naaa", "    check output");
+
+        // Filter
+        // -------------------------------------------------------------------------------------------------------------------------
+        cfgOptionValidSet(cfgOptFilter, true);
+        cfgOptionSet(cfgOptFilter, cfgSourceParam, VARSTRDEF("^aaa$"));
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "filter");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "aaa", "    check output");
+
+        // Subdirectory
+        // -------------------------------------------------------------------------------------------------------------------------
         StringList *argListTmp = strLstDup(argList);
-        strLstAddZ(argListTmp, "--filter=^aaa$");
-        harnessCfgLoad(strLstSize(argListTmp), strLstPtr(argListTmp));
-        TEST_RESULT_STR(strPtr(storageListRender()), "aaa\n", "list files with filter");
-
-        argListTmp = strLstDup(argList);
-        strLstAddZ(argListTmp, "--sort=desc");
-        harnessCfgLoad(strLstSize(argListTmp), strLstPtr(argListTmp));
-        TEST_RESULT_STR(strPtr(storageListRender()), "bbb\naaa\n", "list files with filter and sort");
-
-        storagePutNP(storageNewWriteNP(storageTest, strNew("repo/bbb/ccc")), NULL);
-        argListTmp = strLstDup(argList);
         strLstAddZ(argListTmp, "bbb");
         harnessCfgLoad(strLstSize(argListTmp), strLstPtr(argListTmp));
-        TEST_RESULT_STR(strPtr(storageListRender()), "ccc\n", "list files");
+
+        output = bufNew(0);
+        cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text"));
+        TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "subdirectory");
+        TEST_RESULT_STR(strPtr(strNewBuf(output)), "ccc", "    check output");
 
         // -------------------------------------------------------------------------------------------------------------------------
         // Redirect stdout to a file
@@ -65,6 +149,14 @@ testRun(void)
         dup2(stdoutSave, STDOUT_FILENO);
 
         TEST_RESULT_STR(strPtr(strNewBuf(storageGetNP(storageNewReadNP(storageTest, stdoutFile)))), "ccc\n", "    check text");
+
+        // Too many paths
+        // -------------------------------------------------------------------------------------------------------------------------
+        strLstAddZ(argListTmp, "ccc");
+        harnessCfgLoad(strLstSize(argListTmp), strLstPtr(argListTmp));
+
+        output = bufNew(0);
+        TEST_ERROR(storageListRender(ioBufferWriteNew(output)), ParamInvalidError, "only one path may be specified");
     }
 
     FUNCTION_HARNESS_RESULT_VOID();