From 7b08232049cf9cd2eb350bda327494b1724ff789 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 12 Apr 2023 21:22:44 +0200 Subject: [PATCH] Sanitize logs as appropriate --- core/admin/start.py | 1 - core/base/libs/socrate/socrate/system.py | 49 ++++++++++++++++++++++-- core/dovecot/start.py | 5 +-- core/nginx/start.py | 3 ++ core/postfix/Dockerfile | 2 +- core/postfix/conf/logrotate.conf | 4 -- core/postfix/conf/master.cf | 1 - core/postfix/conf/rsyslog.conf | 43 --------------------- core/postfix/start.py | 16 ++++---- core/rspamd/start.py | 1 - optional/clamav/start.py | 6 +-- optional/radicale/Dockerfile | 2 +- optional/unbound/start.py | 1 - towncrier/newsfragments/2644.misc | 1 + webmails/start.py | 1 - 15 files changed, 64 insertions(+), 72 deletions(-) delete mode 100644 core/postfix/conf/rsyslog.conf create mode 100644 towncrier/newsfragments/2644.misc diff --git a/core/admin/start.py b/core/admin/start.py index f92d845e..be07f84b 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -9,7 +9,6 @@ os.system("chown mailu:mailu -R /dkim") os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") system.drop_privs_to('mailu') -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) system.set_env(['SECRET']) os.system("flask mailu advertise") diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 1e64c5b9..9647cdd1 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,6 +1,8 @@ import hmac import logging as log import os +import sys +import re from pwd import getpwnam import socket import tenacity @@ -24,15 +26,56 @@ def _coerce_value(value): return False return value -def set_env(required_secrets=[]): +class LogFilter(object): + def __init__(self, stream, re_patterns, log_file): + self.stream = stream + if isinstance(re_patterns, list): + self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns])) + elif isinstance(re_patterns, str): + self.pattern = re.compile(re_patterns) + else: + self.pattern = re_patterns + self.found = False + self.log_file = log_file + + def __getattr__(self, attr_name): + return getattr(self.stream, attr_name) + + def write(self, data): + if data == '\n' and self.found: + self.found = False + else: + if not self.pattern.search(data): + self.stream.write(data) + self.stream.flush() + if self.log_file: + try: + with open(self.log_file, 'a', encoding='utf-8') as l: + l.write(data) + except: + pass + else: + # caught bad pattern + self.found = True + + def flush(self): + self.stream.flush() + +def set_env(required_secrets=[], log_filters=[], log_file=None): + if log_filters: + sys.stdout = LogFilter(sys.stdout, log_filters, log_file) + sys.stderr = LogFilter(sys.stderr, log_filters, log_file) + log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING')) + """ This will set all the environment variables and retains only the secrets we need """ - secret_key = os.environ.get('SECRET_KEY') - if not secret_key: + if 'SECRET_KEY_FILE' in os.environ: try: secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip() except Exception as exc: log.error(f"Can't read SECRET_KEY from file: {exc}") raise exc + else: + secret_key = os.environ.get('SECRET_KEY') clean_env() # derive the keys we need for secret in required_secrets: diff --git a/core/dovecot/start.py b/core/dovecot/start.py index fcdc9559..afc0d6f6 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -9,8 +9,7 @@ import sys from podop import run_server from socrate import system, conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -system.set_env() +system.set_env(log_filters=r'waitpid\(\) returned unknown PID \d+$') def start_podop(): system.drop_privs_to('mail') @@ -36,4 +35,4 @@ os.system("chown mail:mail /mail") os.system("chown -R mail:mail /var/lib/dovecot /conf") multiprocessing.Process(target=start_podop).start() -os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) +os.system("dovecot -c /etc/dovecot/dovecot.conf -F") diff --git a/core/nginx/start.py b/core/nginx/start.py index 07932211..5fd4896b 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -2,6 +2,9 @@ import os import subprocess +from socrate import system + +system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$') # Check if a stale pid file exists if os.path.exists("/var/run/nginx.pid"): diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 8565d865..f9a5ac24 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog + ; apk add --no-cache cyrus-sasl-login postfix postfix-pcre logrotate COPY conf/ /conf/ COPY start.py / diff --git a/core/postfix/conf/logrotate.conf b/core/postfix/conf/logrotate.conf index 5882607c..06b1ff29 100644 --- a/core/postfix/conf/logrotate.conf +++ b/core/postfix/conf/logrotate.conf @@ -4,8 +4,4 @@ rotate 52 nocompress extension log create 0644 root root - postrotate - /bin/kill -HUP $(cat /run/rsyslogd.pid) - postfix reload - endscript } diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index f88a19b3..569ea718 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -52,7 +52,6 @@ discard unix - - n - - discard lmtp unix - - n - - lmtp anvil unix - - n - 1 anvil scache unix - - n - 1 scache -postlog unix-dgram n - n - 1 postlogd {# Ensure that the rendered file ends with a newline #} {{- "\n" }} diff --git a/core/postfix/conf/rsyslog.conf b/core/postfix/conf/rsyslog.conf deleted file mode 100644 index 2cda8fb5..00000000 --- a/core/postfix/conf/rsyslog.conf +++ /dev/null @@ -1,43 +0,0 @@ -# rsyslog configuration file -# -# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html -# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html -# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html - - -#### Global directives #### - -# Sets the directory that rsyslog uses for work files. -$WorkDirectory /var/lib/rsyslog - -# Sets default permissions for all log files. -$FileOwner root -$FileGroup adm -$FileCreateMode 0640 -$DirCreateMode 0755 -$Umask 0022 - -# Reduce repeating messages (default off). -$RepeatedMsgReduction on - - -#### Modules #### - -# Provides support for local system logging (e.g. via logger command). -module(load="imuxsock") - -#### Rules #### - -# Discard messages from local test requests -:msg, contains, "connect from localhost[127.0.0.1]" ~ -:msg, contains, "connect from localhost[::1]" ~ -:msg, contains, "haproxy read: short protocol header: QUIT" ~ -:msg, contains, "discarding EHLO keywords: PIPELINING" ~ - -{% if POSTFIX_LOG_FILE %} -# Log mail logs to file -mail.* -{{POSTFIX_LOG_FILE}} -{% endif %} - -# Log mail logs to stdout -mail.* -/dev/stdout diff --git a/core/postfix/start.py b/core/postfix/start.py index 8d8c545f..149e4dae 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -4,15 +4,18 @@ import os import glob import shutil import multiprocessing -import logging as log import sys import re from podop import run_server from socrate import system, conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -system.set_env() +system.set_env(log_filters=[ + r'the Postfix mail system is running\: \d+$', + r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', + r'haproxy read\: short protocol header\: QUIT$', + r'discarding EHLO keywords\: PIPELINING$', + ], log_file=os.environ.get('POSTFIX_LOG_FILE')) os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid") @@ -45,8 +48,6 @@ def is_valid_postconf_line(line): # Actual startup script os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True' -os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local") -os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "") # Postfix requires IPv6 addresses to be wrapped in square brackets if 'RELAYNETS' in os.environ: @@ -86,11 +87,8 @@ if "RELAYUSER" in os.environ: conf.jinja("/conf/sasl_passwd", os.environ, path) os.system("postmap {}".format(path)) -# Configure and start local rsyslog server -conf.jinja("/conf/rsyslog.conf", os.environ, "/etc/rsyslog.conf") -os.system("/usr/sbin/rsyslogd -niNONE &") # Configure logrotate and start crond -if os.environ["POSTFIX_LOG_FILE"] != "": +if os.environ.get('POSTFIX_LOG_FILE'): conf.jinja("/conf/logrotate.conf", os.environ, "/etc/logrotate.d/postfix.conf") os.system("/usr/sbin/crond") if os.path.exists("/overrides/logrotate.conf"): diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 71eeb531..37bb819b 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -9,7 +9,6 @@ import sys import time from socrate import system,conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) system.set_env() # Actual startup script diff --git a/optional/clamav/start.py b/optional/clamav/start.py index 3d0c306d..684d9edd 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import os -import logging as log +import logging as logger import sys +from socrate import system -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -logger=log.getLogger(__name__) +system.set_env(log_filters=r'SelfCheck: Database status OK\.$') # Bootstrap the database if clamav is running for the first time if not os.path.isfile("/data/main.cvd"): diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 904e47db..7ca4d395 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -11,7 +11,7 @@ COPY radicale.conf / RUN echo $VERSION >/version #EXPOSE 5232/tcp -HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 +HEALTHCHECK CMD ["/bin/sh", "-c", "ps ax | grep [/]radicale.conf"] VOLUME ["/data"] diff --git a/optional/unbound/start.py b/optional/unbound/start.py index 5710b6f6..e9e06a48 100755 --- a/optional/unbound/start.py +++ b/optional/unbound/start.py @@ -5,7 +5,6 @@ import logging as log import sys from socrate import conf, system -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) system.set_env() conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf") diff --git a/towncrier/newsfragments/2644.misc b/towncrier/newsfragments/2644.misc new file mode 100644 index 00000000..b77673eb --- /dev/null +++ b/towncrier/newsfragments/2644.misc @@ -0,0 +1 @@ +Filter unwanted logs. diff --git a/webmails/start.py b/webmails/start.py index 09e1a362..c7a1ddf4 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -11,7 +11,6 @@ from socrate import conf, system env = os.environ -logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING")) system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS']) # jinja context