Lock file create and remove improvements (#2183)

* changed the locking function to better support multiple servers running at once and sharing the same config

* helper function testing now runs inside of container

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

target/bin/addmailuser

@@ -42,9 +42,8 @@ PASSWD="${*}"
 [[ -z ${FULL_EMAIL} ]] && { __usage ; errex 'No username specified' ; }
 [[ "${FULL_EMAIL}" =~ .*\@.* ]] || { __usage ; errex 'Username must include the domain' ; }
 
-# Protect config file with lock to avoid race conditions
 touch "${DATABASE}"
-create_lock "$(basename "$0")"
+create_lock # Protect config file with lock to avoid race conditions
 if grep -qi "^$(escape "${FULL_EMAIL}")|" "${DATABASE}"
 then
   echo "User '${FULL_EMAIL}' already exists."

target/bin/delmailuser

@@ -86,7 +86,8 @@ then
   fi
 fi
 
-create_lock "$(basename "$0")"
+create_lock # Protect config file with lock to avoid race conditions
+
 for EMAIL in "${@}"
 do
   ERROR=false

target/bin/setquota

@@ -32,8 +32,7 @@ then
   errex "invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))"
 fi
 
-# Protect config file with lock to avoid race conditions
-create_lock "$(basename "$0")"
+create_lock # Protect config file with lock to avoid race conditions
 
 touch "${DATABASE}"
 if [ -z "${QUOTA}" ]; then

target/bin/updatemailuser

@@ -27,8 +27,7 @@ fi
 
 HASH="$(doveadm pw -s SHA512-CRYPT -u "${USER}" -p "${PASSWD}")"
 
-# Protect config file with lock to avoid race conditions
 touch "${DATABASE}"
-create_lock "$(basename "$0")"
+create_lock # Protect config file with lock to avoid race conditions
 grep -qi "^$(escape "${USER}")|" "${DATABASE}" 2>/dev/null || errex "User \"${USER}\" does not exist"
 sed -i "s ^""${USER}""|.* ""${USER}""|""${HASH}"" " "${DATABASE}"

target/scripts/check-for-changes.sh

@@ -6,8 +6,6 @@
 LOG_DATE=$(date +"%Y-%m-%d %H:%M:%S ")
 _notify 'task' "${LOG_DATE} Start check-for-changes script."
 
-SCRIPT_NAME="$(basename "$0")"
-
 # ? ––––––––––––––––––––––––––––––––––––––––––––– Checks
 
 cd /tmp/docker-mailserver || exit 1
@@ -40,9 +38,6 @@ while true
 do
   LOG_DATE=$(date +"%Y-%m-%d %H:%M:%S ")
 
-  # Lock configuration while working
-  create_lock "${SCRIPT_NAME}"
-
   # get chksum and check it, no need to lock config yet
   _monitored_files_checksums >"${CHKSUM_FILE}.new"
   cmp --silent -- "${CHKSUM_FILE}" "${CHKSUM_FILE}.new"
@@ -53,6 +48,7 @@ do
   if [ $? -eq 1 ]
   then
     _notify 'inf' "${LOG_DATE} Change detected"
+    create_lock # Shared config safety lock
     CHANGED=$(grep -Fxvf "${CHKSUM_FILE}" "${CHKSUM_FILE}.new" | sed 's/^[^ ]\+  //')
 
     # Bug alert! This overwrites the alias set by start-mailserver.sh
@@ -227,11 +223,12 @@ s/$/ regexp:\/etc\/postfix\/regexp/
 
     # prevent restart of dovecot when smtp_only=1
     [[ ${SMTP_ONLY} -ne 1 ]] && supervisorctl restart dovecot
+
+    remove_lock
   fi
 
   # mark changes as applied
   mv "${CHKSUM_FILE}.new" "${CHKSUM_FILE}"
-  remove_lock "${SCRIPT_NAME}"
 
   sleep 1
 done

target/scripts/helper-functions.sh

@@ -1,6 +1,8 @@
 #! /bin/bash
 
 DMS_DEBUG="${DMS_DEBUG:=0}"
+SCRIPT_NAME="$(basename "$0")" # This becomes the sourcing script name (Example: check-for-changes.sh)
+LOCK_ID="$(uuid)" # Used inside of lock files to identify them and prevent removal by other instances of docker-mailserver
 
 # ? --------------------------------------------- BIN HELPER
 
@@ -17,17 +19,33 @@ function escape
 
 function create_lock
 {
-    SCRIPT_NAME="$1"
-    LOCK_FILE="/tmp/docker-mailserver/${SCRIPT_NAME}.lock"
-    [[ -e "${LOCK_FILE}" ]] && errex "Lock file ${LOCK_FILE} exists. Another $1 execution is happening. Try again later."
-    trap remove_lock EXIT # This won't work if the script is, for example, check-for-changes.sh which uses a while loop to stay running; you'll need to include a remove_lock call at the end of your logic
-    touch "${LOCK_FILE}"
+  LOCK_FILE="/tmp/docker-mailserver/${SCRIPT_NAME}.lock"
+  while [[ -e "${LOCK_FILE}" ]]
+  do
+    _notify 'warn' "Lock file ${LOCK_FILE} exists. Another ${SCRIPT_NAME} execution is happening. Trying again shortly..."
+    # Handle stale lock files left behind on crashes
+    # or premature/non-graceful exits of containers while they're making changes
+    if [[ -n "$(find "${LOCK_FILE}" -mmin +1 2>/dev/null)" ]]
+    then
+      _notify 'warn' "Lock file older than 1 minute. Removing stale lock file."
+      rm -f "${LOCK_FILE}"
+      _notify 'inf' "Removed stale lock ${LOCK_FILE}."
+    fi
+    sleep 5
+  done
+  trap remove_lock EXIT
+  echo "${LOCK_ID}" > "${LOCK_FILE}"
 }
 
 function remove_lock
 {
-  SCRIPT_NAME=${SCRIPT_NAME:-$1}
-  rm -f "/tmp/docker-mailserver/${SCRIPT_NAME}.lock"
+  LOCK_FILE="${LOCK_FILE:-"/tmp/docker-mailserver/${SCRIPT_NAME}.lock"}"
+  [[ -z "${LOCK_ID}" ]] && errex "Cannot remove ${LOCK_FILE} as there is no LOCK_ID set"
+  if [[ -e "${LOCK_FILE}" && $(grep -c "${LOCK_ID}" "${LOCK_FILE}") -gt 0 ]] # Ensure we don't delete a lock that's not ours
+  then
+    rm -f "${LOCK_FILE}"
+    _notify 'inf' "Removed lock ${LOCK_FILE}."
+  fi
 }
 
 # ? ––––––––––––––––––––––––––––––––––––––––––––– IP & CIDR