Sanitize logs as appropriate

core/admin/start.py

@@ -9,7 +9,6 @@ os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
 system.drop_privs_to('mailu')
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
 system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")

core/base/libs/socrate/socrate/system.py

@@ -1,6 +1,8 @@
 import hmac
 import logging as log
 import os
+import sys
+import re
 from pwd import getpwnam
 import socket
 import tenacity
@@ -24,15 +26,56 @@ def _coerce_value(value):
         return False
     return value
 
-def set_env(required_secrets=[]):
+class LogFilter(object):
+    def __init__(self, stream, re_patterns, log_file):
+        self.stream = stream
+        if isinstance(re_patterns, list):
+            self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns]))
+        elif isinstance(re_patterns, str):
+            self.pattern = re.compile(re_patterns)
+        else:
+            self.pattern = re_patterns
+        self.found = False
+        self.log_file = log_file
+
+    def __getattr__(self, attr_name):
+        return getattr(self.stream, attr_name)
+
+    def write(self, data):
+        if data == '\n' and self.found:
+            self.found = False
+        else:
+            if not self.pattern.search(data):
+                self.stream.write(data)
+                self.stream.flush()
+                if self.log_file:
+                    try:
+                        with open(self.log_file, 'a', encoding='utf-8') as l:
+                            l.write(data)
+                    except:
+                        pass
+            else:
+                # caught bad pattern
+                self.found = True
+
+    def flush(self):
+        self.stream.flush()
+
+def set_env(required_secrets=[], log_filters=[], log_file=None):
+    if log_filters:
+        sys.stdout = LogFilter(sys.stdout, log_filters, log_file)
+        sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
+    log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
+
     """ This will set all the environment variables and retains only the secrets we need """
-    secret_key = os.environ.get('SECRET_KEY')
-    if not secret_key:
+    if 'SECRET_KEY_FILE' in os.environ:
         try:
             secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
         except Exception as exc:
             log.error(f"Can't read SECRET_KEY from file: {exc}")
             raise exc
+    else:
+        secret_key = os.environ.get('SECRET_KEY')
     clean_env()
     # derive the keys we need
     for secret in required_secrets:

core/dovecot/start.py

@@ -9,8 +9,7 @@ import sys
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=r'waitpid\(\) returned unknown PID \d+$')
 
 def start_podop():
     system.drop_privs_to('mail')
@@ -36,4 +35,4 @@ os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
 
 multiprocessing.Process(target=start_podop).start()
-os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])
+os.system("dovecot -c /etc/dovecot/dovecot.conf -F")

core/nginx/start.py

@@ -2,6 +2,9 @@
 
 import os
 import subprocess
+from socrate import system
+
+system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$')
 
 # Check if a stale pid file exists
 if os.path.exists("/var/run/nginx.pid"):

core/postfix/Dockerfile

@@ -7,7 +7,7 @@ ARG VERSION=local
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog
+  ; apk add --no-cache cyrus-sasl-login postfix postfix-pcre logrotate
 
 COPY conf/ /conf/
 COPY start.py /

core/postfix/conf/logrotate.conf

@@ -4,8 +4,4 @@ rotate 52
 nocompress
 extension log
 create 0644 root root
-      postrotate
-          /bin/kill -HUP $(cat /run/rsyslogd.pid)
-          postfix reload
-      endscript
 }

core/postfix/conf/master.cf

@@ -52,7 +52,6 @@ discard   unix  -       -       n       -       -       discard
 lmtp      unix  -       -       n       -       -       lmtp
 anvil     unix  -       -       n       -       1       anvil
 scache    unix  -       -       n       -       1       scache
-postlog   unix-dgram n  -       n       -       1       postlogd
 
 {# Ensure that the rendered file ends with a newline #}
 {{- "\n" }}

core/postfix/conf/rsyslog.conf

@@ -1,43 +0,0 @@
-# rsyslog configuration file
-#
-# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
-# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
-# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
-
-
-#### Global directives ####
-
-# Sets the directory that rsyslog uses for work files.
-$WorkDirectory /var/lib/rsyslog
-
-# Sets default permissions for all log files.
-$FileOwner root
-$FileGroup adm
-$FileCreateMode 0640
-$DirCreateMode 0755
-$Umask 0022
-
-# Reduce repeating messages (default off).
-$RepeatedMsgReduction on
-
-
-#### Modules ####
-
-# Provides support for local system logging (e.g. via logger command).
-module(load="imuxsock")
-
-#### Rules ####
-
-# Discard messages from local test requests
-:msg, contains, "connect from localhost[127.0.0.1]"  ~
-:msg, contains, "connect from localhost[::1]"  ~
-:msg, contains, "haproxy read: short protocol header: QUIT"  ~
-:msg, contains, "discarding EHLO keywords: PIPELINING"  ~
-
-{% if POSTFIX_LOG_FILE %}
-# Log mail logs to file
-mail.*							-{{POSTFIX_LOG_FILE}}
-{% endif %}
-
-# Log mail logs to stdout
-mail.*							-/dev/stdout

core/postfix/start.py

@@ -4,15 +4,18 @@ import os
 import glob
 import shutil
 import multiprocessing
-import logging as log
 import sys
 import re
 
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=[
+    r'the Postfix mail system is running\: \d+$',
+    r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
+    r'haproxy read\: short protocol header\: QUIT$',
+    r'discarding EHLO keywords\: PIPELINING$',
+    ], log_file=os.environ.get('POSTFIX_LOG_FILE'))
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -45,8 +48,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
-os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 
 # Postfix requires IPv6 addresses to be wrapped in square brackets
 if 'RELAYNETS' in os.environ:
@@ -86,11 +87,8 @@ if "RELAYUSER" in os.environ:
     conf.jinja("/conf/sasl_passwd", os.environ, path)
     os.system("postmap {}".format(path))
 
-# Configure and start local rsyslog server
-conf.jinja("/conf/rsyslog.conf", os.environ, "/etc/rsyslog.conf")
-os.system("/usr/sbin/rsyslogd -niNONE &")
 # Configure logrotate and start crond
-if os.environ["POSTFIX_LOG_FILE"] != "":
+if os.environ.get('POSTFIX_LOG_FILE'):
     conf.jinja("/conf/logrotate.conf", os.environ, "/etc/logrotate.d/postfix.conf")
     os.system("/usr/sbin/crond")
     if os.path.exists("/overrides/logrotate.conf"):

core/rspamd/start.py

@@ -9,7 +9,6 @@ import sys
 import time
 from socrate import system,conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 # Actual startup script

optional/clamav/start.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
 import os
-import logging as log
+import logging as logger
 import sys
+from socrate import system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-logger=log.getLogger(__name__)
+system.set_env(log_filters=r'SelfCheck: Database status OK\.$')
 
 # Bootstrap the database if clamav is running for the first time
 if not os.path.isfile("/data/main.cvd"):

optional/radicale/Dockerfile

@@ -11,7 +11,7 @@ COPY radicale.conf /
 RUN echo $VERSION >/version
 
 #EXPOSE 5232/tcp
-HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1
+HEALTHCHECK CMD ["/bin/sh", "-c", "ps ax | grep [/]radicale.conf"]
 
 VOLUME ["/data"]
 

optional/unbound/start.py

@@ -5,7 +5,6 @@ import logging as log
 import sys
 from socrate import conf, system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")

towncrier/newsfragments/2644.misc

@@ -0,0 +1 @@
+Filter unwanted logs.

webmails/start.py

@@ -11,7 +11,6 @@ from socrate import conf, system
 
 env = os.environ
 
-logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
 system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context