From a4c058d070dfc6f6098fad051a14893ef449eae6 Mon Sep 17 00:00:00 2001
From: David Steele <david@pgmasters.net>
Date: Wed, 17 Jan 2018 15:03:55 -0500
Subject: [PATCH] Add 30 second wait loop to lockAcquire() when fail on no lock
 enabled.

This should help prevent processes that are shutting down from interfering with processes that are starting up.
---
 doc/xml/release.xml           |  4 ++++
 lib/pgBackRest/Common/Lock.pm | 24 ++++++++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 1b2b6fee2..35b448bc9 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -89,6 +89,10 @@
                         <p>Move lock release later in exitSafe() to reduce the chance of a new process starting and acquiring a lock before the old process has exited.</p>
                     </release-item>
 
+                    <release-item>
+                        <p>Add 30 second wait loop to lockAcquire() when fail on no lock enabled.  This should help prevent processes that are shutting down from interfering with processes that are starting up.</p>
+                    </release-item>
+
                     <release-item>
                         <p>Replace <code>cfgCommandTotal()</code>/<code>cfgOptionTotal()</code> functions with constants.  The constants are applicable in more cases and allow the compiler to optimize certain loops more efficiently.</p>
                     </release-item>
diff --git a/lib/pgBackRest/Common/Lock.pm b/lib/pgBackRest/Common/Lock.pm
index 8057a18b3..3690a2552 100644
--- a/lib/pgBackRest/Common/Lock.pm
+++ b/lib/pgBackRest/Common/Lock.pm
@@ -15,6 +15,7 @@ use File::Basename qw(dirname);
 use pgBackRest::Common::Exception;
 use pgBackRest::Common::Log;
 use pgBackRest::Common::String;
+use pgBackRest::Common::Wait;
 use pgBackRest::Config::Config;
 use pgBackRest::Storage::Helper;
 
@@ -91,17 +92,28 @@ sub lockAcquire
         # Create the lock path
         lockPathCreate();
 
-        # Attempt to open the lock file
+        # Loop until the lock can be acquired or timeout.  This is to prevent race conditions where another process is shutting down
+        # but has not yet released its lock.
         $strCurrentLockFile = lockFileName($strLockType, cfgOption(CFGOPT_STANZA, false), $bRemote);
+        my $oWait = waitInit(!defined($bFailOnNoLock) || $bFailOnNoLock ? 30 : 0);
 
-        sysopen($hCurrentLockHandle, $strCurrentLockFile, O_WRONLY | O_CREAT, oct(640))
-            or confess &log(ERROR, "unable to open lock file ${strCurrentLockFile}", ERROR_FILE_OPEN);
+        do
+        {
+            # Attempt to open the lock file
+            next if !sysopen($hCurrentLockHandle, $strCurrentLockFile, O_WRONLY | O_CREAT, oct(640));
+
+            # Attempt to lock the lock file
+            if (!flock($hCurrentLockHandle, LOCK_EX | LOCK_NB))
+            {
+                close($hCurrentLockHandle);
+                undef($hCurrentLockHandle);
+            }
+        }
+        while (!defined($hCurrentLockHandle) && defined($oWait) && waitMore($oWait));
 
         # Attempt to lock the lock file
-        if (!flock($hCurrentLockHandle, LOCK_EX | LOCK_NB))
+        if (!defined($hCurrentLockHandle))
         {
-            close($hCurrentLockHandle);
-
             if (!defined($bFailOnNoLock) || $bFailOnNoLock)
             {
                 confess &log(ERROR, "unable to acquire ${strLockType} lock on file ${strCurrentLockFile}", ERROR_LOCK_ACQUIRE);