diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 6596944b..dc768533 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -36,8 +36,7 @@ def check_credentials(user, password, ip, protocol=None, auth_port=None): if auth_port in WEBMAIL_PORTS or auth_port == '4190' and password.startswith('token-'): if utils.verify_temp_token(user.get_id(), password): is_ok = True - # All tokens are 32 characters hex lowercase - if not is_ok and len(password) == 32: + if not is_ok and utils.is_app_token(password): for token in user.tokens: if (token.check_password(password) and (not token.ip or token.ip == ip)): diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index a72497c4..3d0e227d 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -21,7 +21,8 @@ def nginx_authentication(): utils.limiter.rate_limit_ip(client_ip) return response is_from_webmail = headers['Auth-Port'] in ['10143', '10025'] - if not is_from_webmail and not is_port_25 and utils.limiter.should_rate_limit_ip(client_ip): + is_app_token = utils.is_app_token(headers.get('Auth-Pass','')) + if not is_from_webmail and not is_port_25 and not is_app_token and utils.limiter.should_rate_limit_ip(client_ip): status, code = nginx.get_status(flask.request.headers['Auth-Protocol'], 'ratelimit') response = flask.Response() response.headers['Auth-Status'] = status @@ -36,7 +37,7 @@ def nginx_authentication(): is_valid_user = False username = response.headers.get('Auth-User', None) if response.headers.get("Auth-User-Exists") == "True": - if utils.limiter.should_rate_limit_user(username, client_ip): + if not is_app_token and utils.limiter.should_rate_limit_user(username, client_ip): # FIXME could be done before handle_authentication() status, code = nginx.get_status(flask.request.headers['Auth-Protocol'], 'ratelimit') response = flask.Response() diff --git a/core/admin/mailu/limiter.py b/core/admin/mailu/limiter.py index f02b1662..6fc078c1 100644 --- a/core/admin/mailu/limiter.py +++ b/core/admin/mailu/limiter.py @@ -76,6 +76,7 @@ class LimitWraperFactory(object): return self.storage.incr(f'dedup2-{username}-{truncated_password}', limits.parse(app.config['AUTH_RATELIMIT_USER']).GRANULARITY.seconds, True) limiter.hit(device_cookie if device_cookie_name == username else username) + self.rate_limit_ip(ip, username) """ Device cookies as described on: https://owasp.org/www-community/Slow_Down_Online_Guessing_Attacks_with_Device_Cookies diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py index 42175f05..b2e6027f 100644 --- a/core/admin/mailu/sso/views/base.py +++ b/core/admin/mailu/sso/views/base.py @@ -42,12 +42,13 @@ def login(): destination = app.config['WEB_WEBMAIL'] device_cookie, device_cookie_username = utils.limiter.parse_device_cookie(flask.request.cookies.get('rate_limit')) username = form.email.data - if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip): - flask.flash('Too many attempts from your IP (rate-limit)', 'error') - return flask.render_template('login.html', form=form, fields=fields) - if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username): - flask.flash('Too many attempts for this user (rate-limit)', 'error') - return flask.render_template('login.html', form=form, fields=fields) + if not utils.is_app_token(form.pw.data): + if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip): + flask.flash('Too many attempts from your IP (rate-limit)', 'error') + return flask.render_template('login.html', form=form, fields=fields) + if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username): + flask.flash('Too many attempts for this user (rate-limit)', 'error') + return flask.render_template('login.html', form=form, fields=fields) user = models.User.login(username, form.pw.data) if user: flask.session.regenerate() diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index ced16b7f..5ad684ef 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -15,6 +15,7 @@ import dns.rdataclass import hmac import secrets +import string import time from multiprocessing import Value @@ -525,3 +526,9 @@ def formatCSVField(field): else: data = field.data field.data = ", ".join(data) + +# All tokens are 32 characters hex lowercase +def is_app_token(candidate): + if len(candidate) == 32 and all(c in string.hexdigits[:-6] for c in candidate): + return True + return False diff --git a/core/admin/start.py b/core/admin/start.py index f92d845e..be07f84b 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -9,7 +9,6 @@ os.system("chown mailu:mailu -R /dkim") os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") system.drop_privs_to('mailu') -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) system.set_env(['SECRET']) os.system("flask mailu advertise") diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 1e64c5b9..23cfcdaa 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,6 +1,8 @@ import hmac import logging as log import os +import sys +import re from pwd import getpwnam import socket import tenacity @@ -24,15 +26,72 @@ def _coerce_value(value): return False return value -def set_env(required_secrets=[]): +class LogFilter(object): + def __init__(self, stream, re_patterns, log_file): + self.stream = stream + if isinstance(re_patterns, list): + self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns])) + elif isinstance(re_patterns, str): + self.pattern = re.compile(re_patterns) + else: + self.pattern = re_patterns + self.found = False + self.log_file = log_file + + def __getattr__(self, attr_name): + return getattr(self.stream, attr_name) + + def write(self, data): + if data == '\n' and self.found: + self.found = False + else: + if not self.pattern.search(data): + self.stream.write(data) + self.stream.flush() + if self.log_file: + try: + with open(self.log_file, 'a', encoding='utf-8') as l: + l.write(data) + except: + pass + else: + # caught bad pattern + self.found = True + + def flush(self): + self.stream.flush() + +def _is_compatible_with_hardened_malloc(): + with open('/proc/cpuinfo', 'r') as f: + lines = f.readlines() + for line in lines: + # See #2764, we need vmovdqu + if line.startswith('flags') and ' avx ' not in line: + return False + # See #2541 + if line.startswith('Features') and ' lrcpc ' not in line: + return False + return True + +def set_env(required_secrets=[], log_filters=[], log_file=None): + if log_filters: + sys.stdout = LogFilter(sys.stdout, log_filters, log_file) + sys.stderr = LogFilter(sys.stderr, log_filters, log_file) + log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING')) + + if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc(): + log.warning('Disabling hardened-malloc on this CPU') + del os.environ['LD_PRELOAD'] + """ This will set all the environment variables and retains only the secrets we need """ - secret_key = os.environ.get('SECRET_KEY') - if not secret_key: + if 'SECRET_KEY_FILE' in os.environ: try: secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip() except Exception as exc: log.error(f"Can't read SECRET_KEY from file: {exc}") raise exc + else: + secret_key = os.environ.get('SECRET_KEY') clean_env() # derive the keys we need for secret in required_secrets: diff --git a/core/dovecot/start.py b/core/dovecot/start.py index fcdc9559..afc0d6f6 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -9,8 +9,7 @@ import sys from podop import run_server from socrate import system, conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -system.set_env() +system.set_env(log_filters=r'waitpid\(\) returned unknown PID \d+$') def start_podop(): system.drop_privs_to('mail') @@ -36,4 +35,4 @@ os.system("chown mail:mail /mail") os.system("chown -R mail:mail /var/lib/dovecot /conf") multiprocessing.Process(target=start_podop).start() -os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) +os.system("dovecot -c /etc/dovecot/dovecot.conf -F") diff --git a/core/nginx/start.py b/core/nginx/start.py index 09db7f03..01ecca03 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -2,6 +2,9 @@ import os import subprocess +from socrate import system + +system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$') # Check if a stale pid file exists if os.path.exists("/var/run/nginx.pid"): diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 8565d865..f9a5ac24 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog + ; apk add --no-cache cyrus-sasl-login postfix postfix-pcre logrotate COPY conf/ /conf/ COPY start.py / diff --git a/core/postfix/conf/logrotate.conf b/core/postfix/conf/logrotate.conf index 5882607c..06b1ff29 100644 --- a/core/postfix/conf/logrotate.conf +++ b/core/postfix/conf/logrotate.conf @@ -4,8 +4,4 @@ rotate 52 nocompress extension log create 0644 root root - postrotate - /bin/kill -HUP $(cat /run/rsyslogd.pid) - postfix reload - endscript } diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index f88a19b3..569ea718 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -52,7 +52,6 @@ discard unix - - n - - discard lmtp unix - - n - - lmtp anvil unix - - n - 1 anvil scache unix - - n - 1 scache -postlog unix-dgram n - n - 1 postlogd {# Ensure that the rendered file ends with a newline #} {{- "\n" }} diff --git a/core/postfix/conf/rsyslog.conf b/core/postfix/conf/rsyslog.conf deleted file mode 100644 index 2cda8fb5..00000000 --- a/core/postfix/conf/rsyslog.conf +++ /dev/null @@ -1,43 +0,0 @@ -# rsyslog configuration file -# -# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html -# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html -# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html - - -#### Global directives #### - -# Sets the directory that rsyslog uses for work files. -$WorkDirectory /var/lib/rsyslog - -# Sets default permissions for all log files. -$FileOwner root -$FileGroup adm -$FileCreateMode 0640 -$DirCreateMode 0755 -$Umask 0022 - -# Reduce repeating messages (default off). -$RepeatedMsgReduction on - - -#### Modules #### - -# Provides support for local system logging (e.g. via logger command). -module(load="imuxsock") - -#### Rules #### - -# Discard messages from local test requests -:msg, contains, "connect from localhost[127.0.0.1]" ~ -:msg, contains, "connect from localhost[::1]" ~ -:msg, contains, "haproxy read: short protocol header: QUIT" ~ -:msg, contains, "discarding EHLO keywords: PIPELINING" ~ - -{% if POSTFIX_LOG_FILE %} -# Log mail logs to file -mail.* -{{POSTFIX_LOG_FILE}} -{% endif %} - -# Log mail logs to stdout -mail.* -/dev/stdout diff --git a/core/postfix/start.py b/core/postfix/start.py index 8d8c545f..149e4dae 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -4,15 +4,18 @@ import os import glob import shutil import multiprocessing -import logging as log import sys import re from podop import run_server from socrate import system, conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -system.set_env() +system.set_env(log_filters=[ + r'the Postfix mail system is running\: \d+$', + r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', + r'haproxy read\: short protocol header\: QUIT$', + r'discarding EHLO keywords\: PIPELINING$', + ], log_file=os.environ.get('POSTFIX_LOG_FILE')) os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid") @@ -45,8 +48,6 @@ def is_valid_postconf_line(line): # Actual startup script os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True' -os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local") -os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "") # Postfix requires IPv6 addresses to be wrapped in square brackets if 'RELAYNETS' in os.environ: @@ -86,11 +87,8 @@ if "RELAYUSER" in os.environ: conf.jinja("/conf/sasl_passwd", os.environ, path) os.system("postmap {}".format(path)) -# Configure and start local rsyslog server -conf.jinja("/conf/rsyslog.conf", os.environ, "/etc/rsyslog.conf") -os.system("/usr/sbin/rsyslogd -niNONE &") # Configure logrotate and start crond -if os.environ["POSTFIX_LOG_FILE"] != "": +if os.environ.get('POSTFIX_LOG_FILE'): conf.jinja("/conf/logrotate.conf", os.environ, "/etc/logrotate.d/postfix.conf") os.system("/usr/sbin/crond") if os.path.exists("/overrides/logrotate.conf"): diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 71eeb531..37bb819b 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -9,7 +9,6 @@ import sys import time from socrate import system,conf -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) system.set_env() # Actual startup script diff --git a/docs/antispam.rst b/docs/antispam.rst index 7b0c7373..9b204999 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -83,8 +83,8 @@ Via the multimap filter it is possible to block emails from a sender domain. See The following steps have to be taken to configure an additional symbol (rule) that uses the multimap filter to block emails from sender domain. -1. In the overrides folder create a configuration file for the multimap filter. This configuration is included by Rspamd in the main multimap configuration file. This means you do not have to use the "multimap {}" element. Files in the /mailu/overrides/rspamd/ folder are mapped to /etc/rspamd/override.d. - Create the file /mailu/overrides/rspamd/multimap.conf with contents: +1. In the overrides folder create a configuration file for the multimap filter. This configuration is included by Rspamd in the main multimap configuration file. This means you do not have to use the "multimap {}" element. Files in the ``/mailu/overrides/rspamd/`` folder are mapped to ``/overrides``. + Create the file ``/mailu/overrides/rspamd/multimap.conf`` with contents: .. code-block:: bash @@ -93,7 +93,7 @@ The following steps have to be taken to configure an additional symbol (rule) th local_bl_domain { type = "from"; filter = "email:domain"; - map = "/etc/rspamd/override.d/blacklist.inc"; + map = "/overrides/blacklist.inc"; score = 14; description = "Senders domain part is on the local blacklist"; group = "local_bl"; @@ -129,8 +129,7 @@ The following steps have to be taken to configure an additional symbol (rule) th .. code-block:: bash - docker compose scale antispam=0 - docker compose scale antispam=1 + docker compose up antispam --force-recreate -d 4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed. diff --git a/optional/clamav/start.py b/optional/clamav/start.py index 3d0c306d..684d9edd 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import os -import logging as log +import logging as logger import sys +from socrate import system -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -logger=log.getLogger(__name__) +system.set_env(log_filters=r'SelfCheck: Database status OK\.$') # Bootstrap the database if clamav is running for the first time if not os.path.isfile("/data/main.cvd"): diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 904e47db..7ca4d395 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -11,7 +11,7 @@ COPY radicale.conf / RUN echo $VERSION >/version #EXPOSE 5232/tcp -HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 +HEALTHCHECK CMD ["/bin/sh", "-c", "ps ax | grep [/]radicale.conf"] VOLUME ["/data"] diff --git a/optional/unbound/start.py b/optional/unbound/start.py index 5710b6f6..e9e06a48 100755 --- a/optional/unbound/start.py +++ b/optional/unbound/start.py @@ -5,7 +5,6 @@ import logging as log import sys from socrate import conf, system -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) system.set_env() conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf") diff --git a/towncrier/newsfragments/2644.misc b/towncrier/newsfragments/2644.misc new file mode 100644 index 00000000..3134c97e --- /dev/null +++ b/towncrier/newsfragments/2644.misc @@ -0,0 +1,2 @@ +Filter unwanted logs out. +Disable hardened-malloc if we detect a processor not supporting the AVX extension set diff --git a/towncrier/newsfragments/2772.misc b/towncrier/newsfragments/2772.misc new file mode 100644 index 00000000..b667a131 --- /dev/null +++ b/towncrier/newsfragments/2772.misc @@ -0,0 +1,2 @@ +Always exempt login attempts that use app-tokens from rate-limits +Ensure that unsuccessful login attempts against a valid account hit the ip-based rate-limit too diff --git a/towncrier/newsfragments/2789.bugfix b/towncrier/newsfragments/2789.bugfix new file mode 100644 index 00000000..9fbd1e9f --- /dev/null +++ b/towncrier/newsfragments/2789.bugfix @@ -0,0 +1 @@ +In front, config.py can be called several times. LD_PRELOAD may have already been removed from ENV diff --git a/webmails/start.py b/webmails/start.py index 09e1a362..c7a1ddf4 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -11,7 +11,6 @@ from socrate import conf, system env = os.environ -logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING")) system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS']) # jinja context