Merge #2771

2771: Sanitize logs as appropriate r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

- Sanitize logs as appropriate. 
- change the healthcheck of radicale to something less verbose
- disable hardened-malloc if we detect a processor not supporting the AVX extension set

Should we backport something like that? It could be argued it's a bugfix.

### Related issue(s)
- close #2644 
- close #2764
- #2541

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [ ] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>

core/admin/start.py

@@ -9,7 +9,6 @@ os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
 system.drop_privs_to('mailu')
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
 system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")

core/base/libs/socrate/socrate/system.py

@@ -1,6 +1,8 @@
 import hmac
 import logging as log
 import os
+import sys
+import re
 from pwd import getpwnam
 import socket
 import tenacity
@@ -24,15 +26,68 @@ def _coerce_value(value):
         return False
     return value
 
-def set_env(required_secrets=[]):
+class LogFilter(object):
+    def __init__(self, stream, re_patterns, log_file):
+        self.stream = stream
+        if isinstance(re_patterns, list):
+            self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns]))
+        elif isinstance(re_patterns, str):
+            self.pattern = re.compile(re_patterns)
+        else:
+            self.pattern = re_patterns
+        self.found = False
+        self.log_file = log_file
+
+    def __getattr__(self, attr_name):
+        return getattr(self.stream, attr_name)
+
+    def write(self, data):
+        if data == '\n' and self.found:
+            self.found = False
+        else:
+            if not self.pattern.search(data):
+                self.stream.write(data)
+                self.stream.flush()
+                if self.log_file:
+                    try:
+                        with open(self.log_file, 'a', encoding='utf-8') as l:
+                            l.write(data)
+                    except:
+                        pass
+            else:
+                # caught bad pattern
+                self.found = True
+
+    def flush(self):
+        self.stream.flush()
+
+def _is_compatible_with_hardened_malloc():
+    with open('/proc/cpuinfo', 'r') as f:
+        lines = f.readlines()
+        for line in lines:
+            # See #2764, we need vmovdqu
+            if line.startswith('flags') and ' avx ' not in line:
+                return False
+    return True
+
+def set_env(required_secrets=[], log_filters=[], log_file=None):
+    if log_filters:
+        sys.stdout = LogFilter(sys.stdout, log_filters, log_file)
+        sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
+    log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
+
+    if not _is_compatible_with_hardened_malloc():
+        del os.environ['LD_PRELOAD']
+
     """ This will set all the environment variables and retains only the secrets we need """
-    secret_key = os.environ.get('SECRET_KEY')
-    if not secret_key:
+    if 'SECRET_KEY_FILE' in os.environ:
         try:
             secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
         except Exception as exc:
             log.error(f"Can't read SECRET_KEY from file: {exc}")
             raise exc
+    else:
+        secret_key = os.environ.get('SECRET_KEY')
     clean_env()
     # derive the keys we need
     for secret in required_secrets:

core/dovecot/start.py

@@ -9,8 +9,7 @@ import sys
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=r'waitpid\(\) returned unknown PID \d+$')
 
 def start_podop():
     system.drop_privs_to('mail')
@@ -36,4 +35,4 @@ os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
 
 multiprocessing.Process(target=start_podop).start()
-os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])
+os.system("dovecot -c /etc/dovecot/dovecot.conf -F")

core/nginx/start.py

@@ -2,6 +2,9 @@
 
 import os
 import subprocess
+from socrate import system
+
+system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$')
 
 # Check if a stale pid file exists
 if os.path.exists("/var/run/nginx.pid"):

core/postfix/Dockerfile

@@ -7,7 +7,7 @@ ARG VERSION=local
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog
+  ; apk add --no-cache cyrus-sasl-login postfix postfix-pcre logrotate
 
 COPY conf/ /conf/
 COPY start.py /

core/postfix/conf/logrotate.conf

@@ -4,8 +4,4 @@ rotate 52
 nocompress
 extension log
 create 0644 root root
-      postrotate
-          /bin/kill -HUP $(cat /run/rsyslogd.pid)
-          postfix reload
-      endscript
 }

core/postfix/conf/master.cf

@@ -52,7 +52,6 @@ discard   unix  -       -       n       -       -       discard
 lmtp      unix  -       -       n       -       -       lmtp
 anvil     unix  -       -       n       -       1       anvil
 scache    unix  -       -       n       -       1       scache
-postlog   unix-dgram n  -       n       -       1       postlogd
 
 {# Ensure that the rendered file ends with a newline #}
 {{- "\n" }}

core/postfix/conf/rsyslog.conf

@@ -1,43 +0,0 @@
-# rsyslog configuration file
-#
-# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
-# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
-# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
-
-
-#### Global directives ####
-
-# Sets the directory that rsyslog uses for work files.
-$WorkDirectory /var/lib/rsyslog
-
-# Sets default permissions for all log files.
-$FileOwner root
-$FileGroup adm
-$FileCreateMode 0640
-$DirCreateMode 0755
-$Umask 0022
-
-# Reduce repeating messages (default off).
-$RepeatedMsgReduction on
-
-
-#### Modules ####
-
-# Provides support for local system logging (e.g. via logger command).
-module(load="imuxsock")
-
-#### Rules ####
-
-# Discard messages from local test requests
-:msg, contains, "connect from localhost[127.0.0.1]"  ~
-:msg, contains, "connect from localhost[::1]"  ~
-:msg, contains, "haproxy read: short protocol header: QUIT"  ~
-:msg, contains, "discarding EHLO keywords: PIPELINING"  ~
-
-{% if POSTFIX_LOG_FILE %}
-# Log mail logs to file
-mail.*							-{{POSTFIX_LOG_FILE}}
-{% endif %}
-
-# Log mail logs to stdout
-mail.*							-/dev/stdout

core/postfix/start.py

@@ -4,15 +4,18 @@ import os
 import glob
 import shutil
 import multiprocessing
-import logging as log
 import sys
 import re
 
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=[
+    r'the Postfix mail system is running\: \d+$',
+    r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
+    r'haproxy read\: short protocol header\: QUIT$',
+    r'discarding EHLO keywords\: PIPELINING$',
+    ], log_file=os.environ.get('POSTFIX_LOG_FILE'))
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -45,8 +48,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
-os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 
 # Postfix requires IPv6 addresses to be wrapped in square brackets
 if 'RELAYNETS' in os.environ:
@@ -86,11 +87,8 @@ if "RELAYUSER" in os.environ:
     conf.jinja("/conf/sasl_passwd", os.environ, path)
     os.system("postmap {}".format(path))
 
-# Configure and start local rsyslog server
-conf.jinja("/conf/rsyslog.conf", os.environ, "/etc/rsyslog.conf")
-os.system("/usr/sbin/rsyslogd -niNONE &")
 # Configure logrotate and start crond
-if os.environ["POSTFIX_LOG_FILE"] != "":
+if os.environ.get('POSTFIX_LOG_FILE'):
     conf.jinja("/conf/logrotate.conf", os.environ, "/etc/logrotate.d/postfix.conf")
     os.system("/usr/sbin/crond")
     if os.path.exists("/overrides/logrotate.conf"):

core/rspamd/start.py

@@ -9,7 +9,6 @@ import sys
 import time
 from socrate import system,conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 # Actual startup script

optional/clamav/start.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
 import os
-import logging as log
+import logging as logger
 import sys
+from socrate import system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-logger=log.getLogger(__name__)
+system.set_env(log_filters=r'SelfCheck: Database status OK\.$')
 
 # Bootstrap the database if clamav is running for the first time
 if not os.path.isfile("/data/main.cvd"):

optional/radicale/Dockerfile

@@ -11,7 +11,7 @@ COPY radicale.conf /
 RUN echo $VERSION >/version
 
 #EXPOSE 5232/tcp
-HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1
+HEALTHCHECK CMD ["/bin/sh", "-c", "ps ax | grep [/]radicale.conf"]
 
 VOLUME ["/data"]
 

optional/unbound/start.py

@@ -5,7 +5,6 @@ import logging as log
 import sys
 from socrate import conf, system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")

towncrier/newsfragments/2644.misc

@@ -0,0 +1,2 @@
+Filter unwanted logs out.
+Disable hardened-malloc if we detect a processor not supporting the AVX extension set

webmails/start.py

@@ -11,7 +11,6 @@ from socrate import conf, system
 
 env = os.environ
 
-logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
 system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context