Add backup/expire running status to the info command.

This is implemented by checking for a backup lock on the host where info is running so there are a few limitations:

* It is not currently possible to know which command is running: backup, expire, or stanza-*. The stanza commands are very unlikely to be running so it's pretty safe to guess backup/expire. Command information may be added to the lock file to improve the accuracy of the reported command.

* If the info command is run on a host that is not participating in the backup, e.g. a standby, then there will be no backup lock. This seems like a minor limitation since running info on the repo or primary host is preferred.

doc/xml/release.xml

@@ -13,6 +13,17 @@
 
     <release-list>
         <release date="XXXX-XX-XX" version="2.27dev" title="UNDER DEVELOPMENT">
+            <release-core-list>
+                <release-feature-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                             <release-item-contributor id="stefan.fercot"/>
+                        </release-item-contributor-list>
+
+                        <p>Add <cmd>backup</cmd>/<cmd>expire</cmd> running status to the <cmd>info</cmd> command.</p>
+                    </release-item>
+                </release-feature-list>
+            </release-core-list>
         </release>
 
         <release date="2020-04-20" version="2.26" title="Non-blocking TLS">

doc/xml/user-guide.xml

@@ -1160,6 +1160,8 @@
 
             <p>Each stanza has a separate section and it is possible to limit output to a single stanza with the <br-option>--stanza</br-option> option.  The stanza '<id>status</id>' gives a brief indication of the stanza's health.  If this is '<id>ok</id>' then <backrest/> is functioning normally.  The '<id>wal archive min/max</id>' shows the minimum and maximum WAL currently stored in the archive.  Note that there may be gaps due to archive retention policies or other reasons.</p>
 
+            <p>The '<id>backup/expire running</id>' message will appear beside the '<id>status</id>' information if one of those commands is currently running on the host.</p>
+
             <p>The backups are displayed oldest to newest.  The oldest backup will <i>always</i> be a full backup (indicated by an <id>F</id> at the end of the label) but the newest backup can be full, differential (ends with <id>D</id>), or incremental (ends with <id>I</id>).</p>
 
             <p>The '<id>timestamp start/stop</id>' defines the time period when the backup ran.  The '<id>timestamp stop</id>' can be used to determine the backup to use when performing Point-In-Time Recovery.  More information about Point-In-Time Recovery can be found in the <link section="/pitr">Point-In-Time Recovery</link> section.</p>

src/command/info/info.c

@@ -12,6 +12,7 @@ Info Command
 #include "command/info/info.h"
 #include "common/debug.h"
 #include "common/io/handleWrite.h"
+#include "common/lock.h"
 #include "common/log.h"
 #include "common/memContext.h"
 #include "common/type/json.h"
@@ -63,6 +64,9 @@ VARIANT_STRDEF_STATIC(STANZA_KEY_CIPHER_VAR,                        "cipher");
 VARIANT_STRDEF_STATIC(STANZA_KEY_STATUS_VAR,                        "status");
 VARIANT_STRDEF_STATIC(STANZA_KEY_DB_VAR,                            "db");
 VARIANT_STRDEF_STATIC(STATUS_KEY_CODE_VAR,                          "code");
+VARIANT_STRDEF_STATIC(STATUS_KEY_LOCK_VAR,                          "lock");
+VARIANT_STRDEF_STATIC(STATUS_KEY_LOCK_BACKUP_VAR,                   "backup");
+VARIANT_STRDEF_STATIC(STATUS_KEY_LOCK_BACKUP_HELD_VAR,              "held");
 VARIANT_STRDEF_STATIC(STATUS_KEY_MESSAGE_VAR,                       "message");
 
 #define INFO_STANZA_STATUS_OK                                       "ok"
@@ -77,15 +81,18 @@ STRING_STATIC(INFO_STANZA_STATUS_MESSAGE_NO_BACKUP_STR,             "no valid ba
 #define INFO_STANZA_STATUS_CODE_MISSING_STANZA_DATA                 3
 STRING_STATIC(INFO_STANZA_STATUS_MESSAGE_MISSING_STANZA_DATA_STR,   "missing stanza data");
 
+STRING_STATIC(INFO_STANZA_STATUS_MESSAGE_LOCK_BACKUP_STR,           "backup/expire running");
+
 /***********************************************************************************************************************************
 Set error status code and message for the stanza to the code and message passed.
 ***********************************************************************************************************************************/
 static void
-stanzaStatus(const int code, const String *message, Variant *stanzaInfo)
+stanzaStatus(const int code, const String *message, bool backupLockHeld, Variant *stanzaInfo)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(INT, code);
         FUNCTION_TEST_PARAM(STRING, message);
+        FUNCTION_TEST_PARAM(BOOL, backupLockHeld);
         FUNCTION_TEST_PARAM(VARIANT, stanzaInfo);
     FUNCTION_TEST_END();
 
@@ -98,6 +105,11 @@ stanzaStatus(const int code, const String *message, Variant *stanzaInfo)
     kvAdd(statusKv, STATUS_KEY_CODE_VAR, VARINT(code));
     kvAdd(statusKv, STATUS_KEY_MESSAGE_VAR, VARSTR(message));
 
+    // Construct a specific lock part
+    KeyValue *lockKv = kvPutKv(statusKv, STATUS_KEY_LOCK_VAR);
+    KeyValue *backupLockKv = kvPutKv(lockKv, STATUS_KEY_LOCK_BACKUP_VAR);
+    kvAdd(backupLockKv, STATUS_KEY_LOCK_BACKUP_HELD_VAR, VARBOOL(backupLockHeld));
+
     FUNCTION_TEST_RETURN_VOID();
 }
 
@@ -390,7 +402,7 @@ stanzaInfoList(const String *stanza, StringList *stanzaList, const String *backu
         {
             // If there is no backup.info then set the status to indicate missing
             stanzaStatus(
-                INFO_STANZA_STATUS_CODE_MISSING_STANZA_DATA, INFO_STANZA_STATUS_MESSAGE_MISSING_STANZA_DATA_STR, stanzaInfo);
+                INFO_STANZA_STATUS_CODE_MISSING_STANZA_DATA, INFO_STANZA_STATUS_MESSAGE_MISSING_STANZA_DATA_STR, false, stanzaInfo);
         }
         CATCH(CryptoError)
         {
@@ -444,16 +456,28 @@ stanzaInfoList(const String *stanza, StringList *stanzaList, const String *backu
         kvPut(varKv(stanzaInfo), STANZA_KEY_BACKUP_VAR, varNewVarLst(backupSection));
         kvPut(varKv(stanzaInfo), KEY_ARCHIVE_VAR, varNewVarLst(archiveSection));
 
+        // If a status has not already been set, check if there's a local backup running
+        static bool backupLockHeld = false;
+
+        if (kvGet(varKv(stanzaInfo), STANZA_KEY_STATUS_VAR) == NULL)
+        {
+            // Try to acquire a lock. If not possible, assume another backup or expire is already running.
+            backupLockHeld = !lockAcquire(cfgOptionStr(cfgOptLockPath), stanzaListName, lockTypeBackup, 0, false);
+
+            // Immediately release the lock acquired
+            lockRelease(!backupLockHeld);
+        }
+
         // If a status has not already been set and there are no backups then set status to no backup
         if (kvGet(varKv(stanzaInfo), STANZA_KEY_STATUS_VAR) == NULL &&
             varLstSize(kvGetList(varKv(stanzaInfo), STANZA_KEY_BACKUP_VAR)) == 0)
         {
-            stanzaStatus(INFO_STANZA_STATUS_CODE_NO_BACKUP, INFO_STANZA_STATUS_MESSAGE_NO_BACKUP_STR, stanzaInfo);
+            stanzaStatus(INFO_STANZA_STATUS_CODE_NO_BACKUP, INFO_STANZA_STATUS_MESSAGE_NO_BACKUP_STR, backupLockHeld, stanzaInfo);
         }
 
         // If a status has still not been set then set it to OK
         if (kvGet(varKv(stanzaInfo), STANZA_KEY_STATUS_VAR) == NULL)
-            stanzaStatus(INFO_STANZA_STATUS_CODE_OK, INFO_STANZA_STATUS_MESSAGE_OK_STR, stanzaInfo);
+            stanzaStatus(INFO_STANZA_STATUS_CODE_OK, INFO_STANZA_STATUS_MESSAGE_OK_STR, backupLockHeld, stanzaInfo);
 
         varLstAdd(result, stanzaInfo);
     }
@@ -468,7 +492,8 @@ stanzaInfoList(const String *stanza, StringList *stanzaList, const String *backu
         kvPut(varKv(stanzaInfo), STANZA_KEY_DB_VAR, varNewVarLst(varLstNew()));
         kvPut(varKv(stanzaInfo), STANZA_KEY_BACKUP_VAR, varNewVarLst(varLstNew()));
 
-        stanzaStatus(INFO_STANZA_STATUS_CODE_MISSING_STANZA_PATH, INFO_STANZA_STATUS_MESSAGE_MISSING_STANZA_PATH_STR, stanzaInfo);
+        stanzaStatus(
+            INFO_STANZA_STATUS_CODE_MISSING_STANZA_PATH, INFO_STANZA_STATUS_MESSAGE_MISSING_STANZA_PATH_STR, false, stanzaInfo);
         varLstAdd(result, stanzaInfo);
     }
 
@@ -766,11 +791,26 @@ infoRender(void)
                     KeyValue *stanzaStatus = varKv(kvGet(stanzaInfo, STANZA_KEY_STATUS_VAR));
                     int statusCode = varInt(kvGet(stanzaStatus, STATUS_KEY_CODE_VAR));
 
+                    // Get the lock info
+                    KeyValue *lockKv = varKv(kvGet(stanzaStatus, STATUS_KEY_LOCK_VAR));
+                    KeyValue *backupLockKv = varKv(kvGet(lockKv, STATUS_KEY_LOCK_BACKUP_VAR));
+
                     if (statusCode != INFO_STANZA_STATUS_CODE_OK)
+                    {
+                        // Change displayed status if backup lock is found
+                        if (varBool(kvGet(backupLockKv, STATUS_KEY_LOCK_BACKUP_HELD_VAR)))
+                        {
+                            strCatFmt(
+                                resultStr, "%s (%s, %s)\n", INFO_STANZA_STATUS_ERROR,
+                                strPtr(varStr(kvGet(stanzaStatus, STATUS_KEY_MESSAGE_VAR))),
+                                strPtr(INFO_STANZA_STATUS_MESSAGE_LOCK_BACKUP_STR));
+                        }
+                        else
                         {
                             strCatFmt(
                                 resultStr, "%s (%s)\n", INFO_STANZA_STATUS_ERROR,
                                 strPtr(varStr(kvGet(stanzaStatus, STATUS_KEY_MESSAGE_VAR))));
+                        }
 
                         if (statusCode == INFO_STANZA_STATUS_CODE_MISSING_STANZA_DATA ||
                             statusCode == INFO_STANZA_STATUS_CODE_NO_BACKUP)
@@ -785,8 +825,17 @@ infoRender(void)
 
                         continue;
                     }
+                    else
+                    {
+                        // Change displayed status if backup lock is found
+                        if (varBool(kvGet(backupLockKv, STATUS_KEY_LOCK_BACKUP_HELD_VAR)))
+                        {
+                            strCatFmt(
+                                resultStr, "%s (%s)\n", INFO_STANZA_STATUS_OK, strPtr(INFO_STANZA_STATUS_MESSAGE_LOCK_BACKUP_STR));
+                        }
                         else
                             strCatFmt(resultStr, "%s\n", INFO_STANZA_STATUS_OK);
+                    }
 
                     // Cipher
                     strCatFmt(resultStr, "    cipher: %s\n",

test/src/module/command/infoTest.c

@@ -5,6 +5,7 @@ Test Info Command
 
 #include "common/harnessConfig.h"
 #include "common/harnessInfo.h"
+#include "common/harnessFork.h"
 
 /***********************************************************************************************************************************
 Test Run
@@ -67,6 +68,7 @@ testRun(void)
                     "\"name\":\"stanza1\","
                     "\"status\":{"
                         "\"code\":3,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"missing stanza data\""
                         "}"
                 "}"
@@ -163,6 +165,7 @@ testRun(void)
                      "\"name\":\"stanza1\","
                      "\"status\":{"
                         "\"code\":2,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"no valid backups\""
                     "}"
                 "}"
@@ -180,6 +183,79 @@ testRun(void)
             "        wal archive min/max (9.4-3): none present\n",
             "text - single stanza, no valid backups");
 
+        // Repeat prior tests while a backup lock is held
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD_BEGIN(0, false)
+            {
+                TEST_RESULT_INT_NE(
+                    lockAcquire(cfgOptionStr(cfgOptLockPath), strNew("stanza1"), lockTypeBackup, 0, true), -1,
+                    "create backup/expire lock");
+
+                sleepMSec(1000);
+                lockRelease(true);
+            }
+            HARNESS_FORK_CHILD_END();
+
+            HARNESS_FORK_PARENT_BEGIN()
+            {
+                sleepMSec(250);
+
+                harnessCfgLoad(cfgCmdInfo, argList);
+                TEST_RESULT_STR_Z(
+                    infoRender(),
+                    "["
+                        "{"
+                            "\"archive\":["
+                                "{"
+                                    "\"database\":{"
+                                        "\"id\":2"
+                                    "},"
+                                    "\"id\":\"9.4-3\","
+                                    "\"max\":null,"
+                                    "\"min\":null"
+                                "}"
+                            "],"
+                             "\"backup\":[],"
+                             "\"cipher\":\"aes-256-cbc\","
+                             "\"db\":["
+                                "{"
+                                    "\"id\":1,"
+                                    "\"system-id\":6569239123849665666,"
+                                    "\"version\":\"9.3\""
+                                "},"
+                                "{"
+                                    "\"id\":2,"
+                                    "\"system-id\":6569239123849665679,"
+                                    "\"version\":\"9.4\""
+                                "}"
+                            "],"
+                             "\"name\":\"stanza1\","
+                             "\"status\":{"
+                                "\"code\":2,"
+                                "\"lock\":{\"backup\":{\"held\":true}},"
+                                "\"message\":\"no valid backups\""
+                            "}"
+                        "}"
+                    "]",
+                    "json - single stanza, no valid backups, backup/expire lock detected");
+
+                harnessCfgLoad(cfgCmdInfo, argListText);
+                TEST_RESULT_STR_Z(
+                    infoRender(),
+                    "stanza: stanza1\n"
+                    "    status: error (no valid backups, backup/expire running)\n"
+                    "    cipher: aes-256-cbc\n"
+                    "\n"
+                    "    db (current)\n"
+                    "        wal archive min/max (9.4-3): none present\n",
+                    "text - single stanza, no valid backups, backup/expire lock detected");
+
+            }
+            HARNESS_FORK_PARENT_END();
+        }
+        HARNESS_FORK_END();
+
         // Add WAL segments
         //--------------------------------------------------------------------------------------------------------------------------
         String *archiveDb3 = strNewFmt("%s/9.4-3/0000000100000000", strPtr(archiveStanza1Path));
@@ -331,6 +407,7 @@ testRun(void)
                      "\"name\":\"stanza1\","
                      "\"status\":{"
                         "\"code\":0,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"ok\""
                     "}"
                 "}"
@@ -357,6 +434,130 @@ testRun(void)
             "        wal archive min/max (9.4-3): none present\n",
             "text - single stanza, valid backup, no priors, no archives in latest DB");
 
+        // Repeat prior tests while a backup lock is held
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD_BEGIN(0, false)
+            {
+                TEST_RESULT_INT_NE(
+                    lockAcquire(cfgOptionStr(cfgOptLockPath), strNew("stanza1"), lockTypeBackup, 0, true), -1,
+                    "create backup/expire lock");
+
+                sleepMSec(1000);
+                lockRelease(true);
+            }
+            HARNESS_FORK_CHILD_END();
+
+            HARNESS_FORK_PARENT_BEGIN()
+            {
+                sleepMSec(250);
+
+                harnessCfgLoad(cfgCmdInfo, argList);
+                TEST_RESULT_STR_Z(
+                    infoRender(),
+                    "["
+                        "{"
+                             "\"archive\":["
+                                "{"
+                                    "\"database\":{"
+                                        "\"id\":1"
+                                    "},"
+                                    "\"id\":\"9.4-1\","
+                                    "\"max\":\"000000020000000000000003\","
+                                    "\"min\":\"000000010000000000000002\""
+                                "},"
+                                "{"
+                                    "\"database\":{"
+                                        "\"id\":3"
+                                    "},"
+                                    "\"id\":\"9.4-3\","
+                                    "\"max\":null,"
+                                    "\"min\":null"
+                                "}"
+                            "],"
+                             "\"backup\":["
+                                "{"
+                                    "\"archive\":{"
+                                        "\"start\":null,"
+                                        "\"stop\":null"
+                                    "},"
+                                    "\"backrest\":{"
+                                        "\"format\":5,"
+                                        "\"version\":\"2.04\""
+                                    "},"
+                                    "\"database\":{"
+                                        "\"id\":1"
+                                    "},"
+                                    "\"info\":{"
+                                        "\"delta\":26897030,"
+                                        "\"repository\":{"
+                                            "\"delta\":3159,"
+                                            "\"size\":3159776"
+                                        "},"
+                                        "\"size\":26897030"
+                                    "},"
+                                    "\"label\":\"20181116-154756F\","
+                                    "\"prior\":null,"
+                                    "\"reference\":null,"
+                                    "\"timestamp\":{"
+                                        "\"start\":1542383276,"
+                                        "\"stop\":1542383289"
+                                    "},"
+                                    "\"type\":\"full\""
+                                "}"
+                            "],"
+                             "\"cipher\":\"none\","
+                             "\"db\":["
+                                "{"
+                                    "\"id\":1,"
+                                    "\"system-id\":6569239123849665679,"
+                                    "\"version\":\"9.4\""
+                                "},"
+                                "{"
+                                    "\"id\":2,"
+                                    "\"system-id\":6569239123849665666,"
+                                    "\"version\":\"9.3\""
+                                "},"
+                                "{"
+                                    "\"id\":3,"
+                                    "\"system-id\":6569239123849665679,"
+                                    "\"version\":\"9.4\""
+                                "}"
+                            "],"
+                             "\"name\":\"stanza1\","
+                             "\"status\":{"
+                                "\"code\":0,"
+                                "\"lock\":{\"backup\":{\"held\":true}},"
+                                "\"message\":\"ok\""
+                            "}"
+                        "}"
+                    "]",
+                    "json - single stanza, valid backup, no priors, no archives in latest DB, backup/expire lock detected");
+
+                harnessCfgLoad(cfgCmdInfo, argListText);
+                TEST_RESULT_STR_Z(
+                    infoRender(),
+                    "stanza: stanza1\n"
+                    "    status: ok (backup/expire running)\n"
+                    "    cipher: none\n"
+                    "\n"
+                    "    db (prior)\n"
+                    "        wal archive min/max (9.4-1): 000000010000000000000002/000000020000000000000003\n"
+                    "\n"
+                    "        full backup: 20181116-154756F\n"
+                    "            timestamp start/stop: 2018-11-16 15:47:56 / 2018-11-16 15:48:09\n"
+                    "            wal start/stop: n/a\n"
+                    "            database size: 25.7MB, backup size: 25.7MB\n"
+                    "            repository size: 3MB, repository backup size: 3KB\n"
+                    "\n"
+                    "    db (current)\n"
+                    "        wal archive min/max (9.4-3): none present\n",
+                    "text - single stanza, valid backup, no priors, no archives in latest DB, backup/expire lock detected");
+            }
+            HARNESS_FORK_PARENT_END();
+        }
+        HARNESS_FORK_END();
+
         // backup.info/archive.info files exist, backups exist, archives exist
         //--------------------------------------------------------------------------------------------------------------------------
         content = strNew
@@ -717,6 +918,7 @@ testRun(void)
                      "\"name\":\"stanza1\","
                      "\"status\":{"
                         "\"code\":0,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"ok\""
                     "}"
                 "},"
@@ -743,6 +945,7 @@ testRun(void)
                      "\"name\":\"stanza2\","
                      "\"status\":{"
                         "\"code\":2,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"no valid backups\""
                     "}"
                 "}"
@@ -941,6 +1144,7 @@ testRun(void)
                      "\"name\":\"silly\","
                      "\"status\":{"
                         "\"code\":1,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"missing stanza path\""
                     "}"
                 "}"
@@ -986,6 +1190,7 @@ testRun(void)
                      "\"name\":\"stanza2\","
                      "\"status\":{"
                         "\"code\":2,"
+                        "\"lock\":{\"backup\":{\"held\":false}},"
                         "\"message\":\"no valid backups\""
                     "}"
                 "}"