[Compose, DockerAPI, Web, Watchdog] Watchdog may send notification mails (todo: docs), DockerAPI via Flesk for limited access

2024-12-25 02:29:32 +02:00 · 2017-10-05 23:38:33 +02:00 · 2017-10-05 23:38:33 +02:00 · fc18d153cd
commit fc18d153cd
parent 9860d44d04
9 changed files with 152 additions and 22 deletions
--- a/data/Dockerfiles/dockerapi/Dockerfile
+++ b/data/Dockerfiles/dockerapi/Dockerfile
@ -0,0 +1,8 @@
+FROM python:2-alpine
+LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+
+RUN apk add -U --no-cache iptables ip6tables
+RUN pip install docker flask flask-restful
+
+COPY server.py /
+CMD ["python2", "-u", "/server.py"]
--- a/data/Dockerfiles/dockerapi/server.py
+++ b/data/Dockerfiles/dockerapi/server.py
@ -0,0 +1,41 @@
+from flask import Flask
+from flask_restful import Resource, Api
+from docker import APIClient
+
+dockercli = APIClient(base_url='unix://var/run/docker.sock')
+app = Flask(__name__)
+api = Api(app)
+
+class Containers(Resource):
+    def get(self):
+        return dockercli.containers(all=True)
+
+class ContainerInfo(Resource):
+    def get(self, container_id):
+        return dockercli.containers(all=True, filters={"id": container_id})
+
+class ContainerStart(Resource):
+    def post(self, container_id):
+        try:
+            dockercli.start(container_id);
+        except:
+            return 'Error'
+        else:
+            return 'OK'
+
+class ContainerStop(Resource):
+    def post(self, container_id):
+        try:
+            dockercli.stop(container_id);
+        except: 
+            return 'Error'
+        else:
+            return 'OK'
+
+api.add_resource(Containers, '/info/container/all')
+api.add_resource(ContainerInfo, '/info/container/<string:container_id>')
+api.add_resource(ContainerStop, '/stop/container/<string:container_id>')
+api.add_resource(ContainerStart, '/start/container/<string:container_id>')
+
+if __name__ == '__main__':
+    app.run(debug=False, host='0.0.0.0', port='8080')
--- a/data/Dockerfiles/watchdog/Dockerfile
+++ b/data/Dockerfiles/watchdog/Dockerfile
@ -16,7 +16,15 @@ RUN apk add --update \
 	nagios-plugins-dns \
    nagios-plugins-disk \
 	bind-tools \
-	redis
+	redis \
+	perl \
+	perl-io-socket-ssl \
+	perl-socket \
+	perl-socket6 \
+	perl-mime-lite \
+	perl-term-readkey \
+	&& curl https://raw.githubusercontent.com/mludvig/smtp-cli/v3.8/smtp-cli -o /smtp-cli \
+	&& chmod +x smtp-cli

 COPY watchdog.sh /watchdog.sh

--- a/data/Dockerfiles/watchdog/watchdog.sh
+++ b/data/Dockerfiles/watchdog/watchdog.sh
@ -38,6 +38,26 @@ log_to_redis() {
 	redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}"
 }

+function mail_error() {
+  [[ -z ${1} ]] && return 1
+  [[ -z ${2} ]] && return 2
+  RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'})
+  RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
+  if [[ -z ${RCPT_MX} ]]; then
+    log_to_redis "Cannot determine MX for ${1}, skipping email notification..."
+    echo "Cannot determine MX for ${1}"
+    return 1
+  fi
+  ./smtp-cli --missing-modules-ok \
+    --subject="Watchdog: ${2} service hit the error rate limit" \
+    --body-plain="Service was restarted, please check your mailcow installation." \
+    --to=${1} \
+    --from="watchdog@${MAILCOW_HOSTNAME}" \
+    --server="${RCPT_MX}" \
+    --hello-host=${MAILCOW_HOSTNAME}
+}
+
+
 get_container_ip() {
  # ${1} is container
  CONTAINER_ID=
@ -233,6 +253,7 @@ dns_checks() {
 while true; do
  if ! nginx_checks; then
    log_to_redis "Nginx hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "nginx-mailcow"
    echo -e "\e[31m$(date) - Nginx hit error limit\e[0m"
    echo nginx-mailcow > /tmp/com_pipe
  fi
@ -244,6 +265,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! mysql_checks; then
    log_to_redis "MySQL hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "mysql-mailcow"
    echo -e "\e[31m$(date) - MySQL hit error limit\e[0m"
    echo mysql-mailcow > /tmp/com_pipe
  fi
@ -255,6 +277,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! phpfpm_checks; then
    log_to_redis "PHP-FPM hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "php-fpm-mailcow"
    echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m"
    echo php-fpm-mailcow > /tmp/com_pipe
  fi
@ -266,6 +289,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! sogo_checks; then
    log_to_redis "SOGo hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "sogo-mailcow"
    echo -e "\e[31m$(date) - SOGo hit error limit\e[0m"
    echo sogo-mailcow > /tmp/com_pipe
  fi
@ -277,6 +301,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! postfix_checks; then
    log_to_redis "Postfix hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "postfix-mailcow"
    echo -e "\e[31m$(date) - Postfix hit error limit\e[0m"
    echo postfix-mailcow > /tmp/com_pipe
  fi
@ -288,6 +313,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! dovecot_checks; then
    log_to_redis "Dovecot hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "dovecot-mailcow"
    echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m"
    echo dovecot-mailcow > /tmp/com_pipe
  fi
@ -299,6 +325,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! dns_checks; then
    log_to_redis "Unbound hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "unbound-mailcow"
    echo -e "\e[31m$(date) - Unbound hit error limit\e[0m"
    #echo unbound-mailcow > /tmp/com_pipe
  fi
@ -310,6 +337,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  if ! rspamd_checks; then
    log_to_redis "Rspamd hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "rspamd-mailcow"
    echo -e "\e[31m$(date) - Rspamd hit error limit\e[0m"
    echo rspamd-mailcow > /tmp/com_pipe
  fi
--- a/data/conf/rspamd/local.d/multimap.conf
+++ b/data/conf/rspamd/local.d/multimap.conf
@ -23,7 +23,7 @@ KEEP_SPAM {

 SPOOFED_SENDER {
  type = "rcpt";
-  filter = "email:domain";
+  filter = "email:domain:tld";
  map = "redis://DOMAIN_MAP";
  require_symbols = "AUTH_NA | !RCVD_VIA_SMTP_AUTH";
 }
--- a/data/web/api.php
+++ b/data/web/api.php
@ -0,0 +1,32 @@
+<?php
+set_time_limit (0);
+
+$address = '0.0.0.0';
+
+$port = 7777;
+$con = 1;
+$word = "";
+
+$sock = socket_create(AF_INET, SOCK_STREAM, 0);
+$bind = socket_bind($sock, $address, $port);
+
+socket_listen($sock);
+
+while ($con == 1)
+{
+    $client = socket_accept($sock);
+    $input = socket_read($client, 2024);
+
+    if ($input == 'exit') 
+    {
+        $close = socket_close($sock);
+        $con = 0;
+    }
+
+    if($con == 1)
+    {
+        $word .= $input;
+    }
+}
+
+echo $word;
--- a/data/web/inc/call_sogo_ctrl.php
+++ b/data/web/inc/call_sogo_ctrl.php
@ -11,8 +11,7 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
  curl_setopt($curl, CURLOPT_HTTPHEADER,array( 'Content-Type: application/json' ));
  switch($action) {
    case 'get_id':
-      curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
-      curl_setopt($curl, CURLOPT_URL, 'http:/v1.26/containers/json?all=1');
+      curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/info/container/all');
      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
      curl_setopt($curl, CURLOPT_POST, 0);
      $response = curl_exec($curl);
@ -35,13 +34,11 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
      return false;
    break;
    case 'info':
-      curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
      $container_id = docker($service_name, 'get_id');
      if (ctype_xdigit($container_id)) {
-        curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
-        curl_setopt($curl, CURLOPT_URL, 'http/containers/' . $container_id . '/json');
-        curl_setopt($curl, CURLOPT_POST, 0);
+        curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/info/container/' . $container_id);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($curl, CURLOPT_POST, 0);
        $response = curl_exec($curl);
        if ($response === false) {
          $err = curl_error($curl);
@ -65,9 +62,8 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
    case 'post':
      if (!empty($post_action)) {
        $container_id = docker($service_name, 'get_id');
-        if (ctype_xdigit($container_id)) {
-          curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
-          curl_setopt($curl, CURLOPT_URL, 'http/containers/' . $container_id . '/' . $post_action);
+        if (ctype_xdigit($container_id) && ctype_alnum($post_action)) {
+          curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/' . $post_action . '/container/' . $container_id);
          curl_setopt($curl, CURLOPT_POST, 1);
          if (!empty($post_fields)) {
            curl_setopt( $curl, CURLOPT_POSTFIELDS, json_encode($post_fields));
@ -96,30 +92,30 @@ function docker($service_name, $action, $post_action = null, $post_fields = null

 if ($_GET['ACTION'] == "start") {
  $retry = 0;
-  while (!docker('sogo-mailcow', 'info')['State']['Running'] && $retry <= 3) {
+  while (docker('sogo-mailcow', 'info')[0]['State'] != "running" && $retry <= 3) {
    $response = docker('sogo-mailcow', 'post', 'start');
-    $last_response = ($response === true) ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-warning">Error: ' . $response . '</span></b>';
-    if ($response === true) {
+    $last_response = (trim($response) == "\"OK\"") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response . '</span></b>';
+    if (trim($response) == "\"OK\"") {
      break;
    }
    usleep(1500000);
    $retry++;
  }
-  echo $last_response;
+  echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
 }

 if ($_GET['ACTION'] == "stop") {
  $retry = 0;
-  while (docker('sogo-mailcow', 'info')['State']['Running'] && $retry <= 3) {
+  while (docker('sogo-mailcow', 'info')[0]['State'] == "running" && $retry <= 3) {
    $response = docker('sogo-mailcow', 'post', 'stop');
-    $last_response = ($response === true) ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-warning">Error: ' . $response . '</span></b>';
-    if ($response === true) {
+    $last_response = (trim($response) == "\"OK\"") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response . '</span></b>';
+    if (trim($response) == "\"OK\"") {
      break;
    }
    usleep(1500000);
    $retry++;
  }
-  echo $last_response;
+  echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
 }

 ?>
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -105,7 +105,6 @@ services:
        - ./data/web:/web:ro
        - ./data/conf/rspamd/dynmaps:/dynmaps:ro
        - dkim-vol-1:/data/dkim
-        - /var/run/docker.sock:/var/run/docker.sock:ro
      environment:
        - DBNAME=${DBNAME}
        - DBUSER=${DBUSER}
@ -297,7 +296,7 @@ services:
        - /lib/modules:/lib/modules:ro

    watchdog-mailcow:
-      image: mailcow/watchdog:1.3
+      image: mailcow/watchdog:1.4
      build: ./data/Dockerfiles/watchdog
      init: false
      volumes:
@ -309,11 +308,24 @@ services:
        - DBUSER=${DBUSER}
        - DBPASS=${DBPASS}
        - USE_WATCHDOG=${USE_WATCHDOG:-n}
+        - WATCHDOG_NOTIFY_EMAIL=${WATCHDOG_NOTIFY_EMAIL}
+        - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
      networks:
        mailcow-network:
          aliases:
            - watchdog

+    dockerapi-mailcow:
+      image: mailcow/dockerapi:1.0
+      build: ./data/Dockerfiles/dockerapi
+      init: false
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock:ro
+      networks:
+        mailcow-network:
+          aliases:
+            - dockerapi
+
    ipv6nat:
      image: robbertkl/ipv6nat
      restart: always
--- a/update.sh
+++ b/update.sh
@ -4,7 +4,7 @@ for bin in curl docker-compose docker git awk sha1sum; do
 	if [[ -z $(which ${bin}) ]]; then echo "Cannot find ${bin}, exiting..."; exit 1; fi
 done

-CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "USE_WATCHDOG" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT")
+CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "USE_WATCHDOG" "WATCHDOG_NOTIFY_EMAIL" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT")
 echo >> mailcow.conf
 for option in ${CONFIG_ARRAY[@]}; do
 	if [[ ${option} == "ADDITIONAL_SAN" ]]; then
@ -22,6 +22,11 @@ for option in ${CONFIG_ARRAY[@]}; do
 			echo "Adding new option \"${option}\" to mailcow.conf"
 			echo "DOVEADM_PORT=127.0.0.1:19991" >> mailcow.conf
 		fi
+	elif [[ ${option} == "WATCHDOG_NOTIFY_EMAIL" ]]; then
+		if ! grep -q ${option} mailcow.conf; then
+			echo "Adding new option \"${option}\" to mailcow.conf"
+			echo "WATCHDOG_NOTIFY_EMAIL=" >> mailcow.conf
+		fi
 	elif ! grep -q ${option} mailcow.conf; then
 		echo "Adding new option \"${option}\" to mailcow.conf"
 		echo "${option}=n" >> mailcow.conf