Merge #2562

2562: Dynamic address resolution everywhere r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

Use dynamic address resolution everywhere.
Derive a new key for admin/SECRET_KEY
Cleanup the environment

This should allow restarting containers.

### Related issue(s)
- closes #1341
- closes #1013
- closes #1430

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [x] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>

core/admin/mailu/configuration.py

@@ -1,7 +1,6 @@
 import os
 
 from datetime import timedelta
-from socrate import system
 import ipaddress
 
 DEFAULT_CONFIG = {
@@ -83,17 +82,6 @@ DEFAULT_CONFIG = {
     'PROXY_AUTH_WHITELIST': '',
     'PROXY_AUTH_HEADER': 'X-Auth-Email',
     'PROXY_AUTH_CREATE': False,
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_LMTP': 'imap:2525',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_AUTHSMTP': 'smtp',
-    'HOST_ADMIN': 'admin',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_WEBDAV': 'webdav:5232',
-    'HOST_REDIS': 'redis',
-    'HOST_FRONT': 'front',
     'SUBNET': '192.168.203.0/24',
     'SUBNET6': None
 }
@@ -111,19 +99,6 @@ class ConfigManager:
     def __init__(self):
         self.config = dict()
 
-    def get_host_address(self, name):
-        # if MYSERVICE_ADDRESS is defined, use this
-        if f'{name}_ADDRESS' in os.environ:
-            return os.environ.get(f'{name}_ADDRESS')
-        # otherwise use the host name and resolve it
-        return system.resolve_address(self.config[f'HOST_{name}'])
-
-    def resolve_hosts(self):
-        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
-            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
-        if self.config['WEBMAIL'] != 'none':
-            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
-
     def __get_env(self, key, value):
         key_file = key + "_FILE"
         if key_file in os.environ:
@@ -144,11 +119,14 @@ class ConfigManager:
         # get current app config
         self.config.update(app.config)
         # get environment variables
+        for key in os.environ:
+            if key.endswith('_ADDRESS'):
+                self.config[key] = os.environ[key]
+
         self.config.update({
             key: self.__coerce_value(self.__get_env(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
-        self.resolve_hosts()
 
         # automatically set the sqlalchemy string
         if self.config['DB_FLAVOR']:

core/admin/mailu/internal/nginx.py

@@ -2,7 +2,6 @@ from mailu import models, utils
 from flask import current_app as app
 from socrate import system
 
-import re
 import urllib
 import ipaddress
 import sqlalchemy.exc
@@ -128,20 +127,16 @@ def get_status(protocol, status):
     status, codes = STATUSES[status]
     return status, codes[protocol]
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
 def get_server(protocol, authenticated=False):
     if protocol == "imap":
-        hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143)
+        hostname, port = app.config['IMAP_ADDRESS'], 143
     elif protocol == "pop3":
-        hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110)
+        hostname, port = app.config['IMAP_ADDRESS'], 110
     elif protocol == "smtp":
         if authenticated:
-            hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
+            hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
-            hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+            hostname, port = app.config['SMTP_ADDRESS'], 25
     try:
         # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)

core/admin/mailu/models.py

@@ -421,8 +421,7 @@ class Email(object):
         """ send an email to the address """
         try:
             f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-            ip, port = app.config['HOST_LMTP'].rsplit(':')
-            with smtplib.LMTP(ip, port=port) as lmtp:
+            with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
                 to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
                 msg = text.MIMEText(body)
                 msg['Subject'] = subject

core/admin/mailu/ui/templates/client.html

@@ -21,7 +21,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
@@ -46,7 +46,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>

core/admin/run_dev.sh

@@ -75,12 +75,15 @@ ENV \
     DEBUG_ASSETS="/app/static" \
     DEBUG_TB_INTERCEPT_REDIRECTS=False \
     \
-    IMAP_ADDRESS="127.0.0.1" \
-    POP3_ADDRESS="127.0.0.1" \
-    AUTHSMTP_ADDRESS="127.0.0.1" \
+    ADMIN_ADDRESS="127.0.0.1" \
+    FRONT_ADDRESS="127.0.0.1" \
     SMTP_ADDRESS="127.0.0.1" \
+    IMAP_ADDRESS="127.0.0.1" \
     REDIS_ADDRESS="127.0.0.1" \
-    WEBMAIL_ADDRESS="127.0.0.1"
+    ANTIVIRUS_ADDRESS="127.0.0.1" \
+    ANTISPAM_ADDRESS="127.0.0.1" \
+    WEBMAIL_ADDRESS="127.0.0.1" \
+    WEBDAV_ADDRESS="127.0.0.1"
 
 CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
 EOF

core/admin/start.py

@@ -4,6 +4,7 @@ import os
 import logging as log
 from pwd import getpwnam
 import sys
+from socrate import system
 
 os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
@@ -12,6 +13,7 @@ os.setgid(mailu_id.pw_gid)
 os.setuid(mailu_id.pw_uid)
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
+system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")
 os.system("flask db upgrade")

core/base/Dockerfile

@@ -17,11 +17,21 @@ RUN set -euxo pipefail \
   ; ! [[ "${machine}" == x86_64 ]] \
     || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
 
-ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
-ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
-ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
+ENV \
+  LD_PRELOAD="/usr/lib/libhardened_malloc.so" \
+  CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \
+  LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \
+  ADMIN_ADDRESS="admin" \
+  FRONT_ADDRESS="front" \
+  SMTP_ADDRESS="smtp" \
+  IMAP_ADDRESS="imap" \
+  REDIS_ADDRESS="redis" \
+  ANTIVIRUS_ADDRESS="antivirus" \
+  ANTISPAM_ADDRESS="antispam" \
+  WEBMAIL_ADDRESS="webmail" \
+  WEBDAV_ADDRESS="webdav"
 
 WORKDIR /app
 

core/base/libs/socrate/socrate/system.py

@@ -1,7 +1,8 @@
+import hmac
+import logging as log
+import os
 import socket
 import tenacity
-from os import environ
-import logging as log
 
 @tenacity.retry(stop=tenacity.stop_after_attempt(100),
                 wait=tenacity.wait_random(min=2, max=5))
@@ -15,24 +16,32 @@ def resolve_hostname(hostname):
         log.warn("Unable to lookup '%s': %s",hostname,e)
         raise e
 
+def _coerce_value(value):
+    if isinstance(value, str) and value.lower() in ('true','yes'):
+        return True
+    elif isinstance(value, str) and value.lower() in ('false', 'no'):
+        return False
+    return value
 
-def resolve_address(address):
-    """ This function is identical to ``resolve_hostname`` but also supports
-    resolving an address, i.e. including a port.
-    """
-    hostname, *rest = address.rsplit(":", 1)
-    ip_address = resolve_hostname(hostname)
-    if ":" in ip_address:
-        ip_address = "[{}]".format(ip_address)
-    return ip_address + "".join(":" + port for port in rest)
+def set_env(required_secrets=[]):
+    """ This will set all the environment variables and retains only the secrets we need """
+    secret_key = os.environ.get('SECRET_KEY')
+    if not secret_key:
+        try:
+            secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
+        except Exception as exc:
+            log.error(f"Can't read SECRET_KEY from file: {exc}")
+            raise exc
+    clean_env()
+    # derive the keys we need
+    for secret in required_secrets:
+        os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
+    return {
+            key: _coerce_value(os.environ.get(key, value))
+            for key, value in os.environ.items()
+           }
 
-def get_host_address_from_environment(name, default):
-    """ This function looks up an envionment variable ``{{ name }}_ADDRESS``.
-    If it's defined, it is returned unmodified. If it's undefined, an environment
-    variable ``HOST_{{ name }}`` is looked up and resolved to an ip address.
-    If this is also not defined, the default is resolved to an ip address.
-    """
-    if "{}_ADDRESS".format(name) in environ:
-        return environ.get("{}_ADDRESS".format(name))
-    return resolve_address(environ.get("HOST_{}".format(name), default))
+def clean_env():
+    """ remove all secret keys """
+    [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]

core/base/libs/socrate/test.py

@@ -78,40 +78,5 @@ class TestSystem(unittest.TestCase):
             "2001:db8::f00"
         )
 
-
-    def test_resolve_address(self):
-        self.assertEqual(
-            system.resolve_address("1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        self.assertEqual(
-            system.resolve_address("2001-db8--f00.sslip.io:80"),
-            "[2001:db8::f00]:80"
-        )
-
-    def test_get_host_address_from_environment(self):
-        if "TEST_ADDRESS" in os.environ:
-            del os.environ["TEST_ADDRESS"]
-        if "HOST_TEST" in os.environ:
-            del os.environ["HOST_TEST"]
-        # if nothing is set, the default must be resolved
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        # if HOST is set, the HOST must be resolved
-        os.environ['HOST_TEST']="1.2.3.5.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.5:80"
-        )
-        # if ADDRESS is set, the ADDRESS must be returned unresolved
-        os.environ['TEST_ADDRESS']="1.2.3.6.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.6.sslip.io:80"
-        )
-
-
 if __name__ == "__main__":
     unittest.main()

core/dovecot/conf/ham.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/conf/spam.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/start.py

@@ -11,6 +11,7 @@ from podop import run_server
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 def start_podop():
     id_mail = getpwnam('mail')
@@ -24,10 +25,6 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-
 for dovecot_file in glob.glob("/conf/*.conf"):
     conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 

core/nginx/conf/nginx.conf

@@ -77,12 +77,12 @@ http {
       root /static;
       # Variables for proxifying
       set $admin {{ ADMIN_ADDRESS }};
-      set $antispam {{ ANTISPAM_WEBUI_ADDRESS }};
+      set $antispam {{ ANTISPAM_ADDRESS }}:11334;
       {% if WEBMAIL_ADDRESS %}
       set $webmail {{ WEBMAIL_ADDRESS }};
       {% endif %}
       {% if WEBDAV_ADDRESS %}
-      set $webdav {{ WEBDAV_ADDRESS }};
+      set $webdav {{ WEBDAV_ADDRESS }}:5232;
       {% endif %}
       client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
 

core/nginx/config.py

@@ -5,8 +5,8 @@ import logging as log
 import sys
 from socrate import system, conf
 
+system.set_env()
 args = os.environ.copy()
-
 log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
 
 args['TLS_PERMISSIVE'] = str(args.get('TLS_PERMISSIVE')).lower() not in ('false', 'no')
@@ -17,13 +17,6 @@ with open("/etc/resolv.conf") as handle:
     resolver = content[content.index("nameserver") + 1]
     args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver
 
-args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-args["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-if args["WEBMAIL"] != "none":
-    args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
-if args["WEBDAV"] != "none":
-    args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232")
-
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")
 keypair_name = os.getenv("TLS_KEYPAIR_FILENAME", default="key.pem")

core/postfix/conf/main.cf

@@ -81,7 +81,7 @@ virtual_mailbox_maps = ${podop}mailbox
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
 transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
-virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }}
+virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525
 
 # Sender and recipient canonical maps, mostly for SRS
 sender_canonical_maps = ${podop}sendermap
@@ -126,7 +126,7 @@ unverified_recipient_reject_reason = Address lookup failure
 # Milter
 ###############
 
-smtpd_milters = inet:{{ ANTISPAM_MILTER_ADDRESS }}
+smtpd_milters = inet:{{ ANTISPAM_ADDRESS }}:11332
 milter_protocol = 6
 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
 milter_default_action = tempfail

core/postfix/start.py

@@ -13,6 +13,7 @@ from pwd import getpwnam
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -45,10 +46,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
-os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
 os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
 os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 

core/rspamd/conf/antivirus.conf

@@ -3,7 +3,7 @@ clamav {
   scan_mime_parts = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}";
+  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
   {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
   action = "reject"
   {% endif %}

core/rspamd/start.py

@@ -6,18 +6,13 @@ import logging as log
 import requests
 import sys
 import time
-from socrate import system, conf
+from socrate import system,conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 # Actual startup script
 
-os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-
-if os.environ.get("ANTIVIRUS") == 'clamav':
-    os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310")
-
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
 

docs/configuration.rst

@@ -249,32 +249,22 @@ virus mails during SMTP dialogue, so the sender will receive a reject message.
 Infrastructure settings
 -----------------------
 
-Various environment variables ``HOST_*`` can be used to run Mailu containers
+Various environment variables ``*_ADDRESS`` can be used to run Mailu containers
 separately from a supported orchestrator. It is used by the various components
-to find the location of the other containers it depends on. They can contain an
-optional port number. Those variables are:
+to find the location of the other containers it depends on. Those variables are:
 
-- ``HOST_IMAP``: the container that is running the IMAP server (default: ``imap``, port 143)
-- ``HOST_LMTP``: the container that is running the LMTP server (default: ``imap:2525``)
-- ``HOST_HOSTIMAP``: the container that is running the IMAP server for the webmail (default: ``imap``, port 10143)
-- ``HOST_POP3``: the container that is running the POP3 server (default: ``imap``, port 110)
-- ``HOST_SMTP``: the container that is running the SMTP server (default: ``smtp``, port 25)
-- ``HOST_AUTHSMTP``: the container that is running the authenticated SMTP server for the webnmail (default: ``smtp``, port 10025)
-- ``HOST_ADMIN``: the container that is running the admin interface (default: ``admin``)
-- ``HOST_ANTISPAM_MILTER``: the container that is running the antispam milter service (default: ``antispam:11332``)
-- ``HOST_ANTISPAM_WEBUI``: the container that is running the antispam webui service (default: ``antispam:11334``)
-- ``HOST_ANTIVIRUS``: the container that is running the antivirus service (default: ``antivirus:3310``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
-- ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``)
-- ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
+- ``ADMIN_ADDRESS``
+- ``ANTISPAM_ADDRESS``
+- ``ANTIVIRUS_ADDRESS``
+- ``FRONT_ADDRESS``
+- ``IMAP_ADDRESS``
+- ``REDIS_ADDRESS``
+- ``SMTP_ADDRESS``
+- ``WEBDAV_ADDRESS``
+- ``WEBMAIL_ADDRESS``
 
-The startup scripts will resolve ``HOST_*`` to their IP addresses and store the result in ``*_ADDRESS`` for further use.
-
-Alternatively, ``*_ADDRESS`` can directly be set. In this case, the values of ``*_ADDRESS`` is kept and not
-resolved. This can be used to rely on DNS based service discovery with changing services IP addresses.
-When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to
-resolve the hostnames.
+These are used for DNS based service discovery with possibly changing services IP addresses.
+``*_ADDRESS`` values must be fully qualified domain names without port numbers.
 
 .. _db_settings:
 

optional/fetchmail/fetchmail.py

@@ -7,7 +7,6 @@ from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
-import re
 import requests
 from socrate import system
 import sys
@@ -34,11 +33,6 @@ poll "{host}" proto {protocol}  port {port}
 """
 
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
-
 def escape_rc_string(arg):
     return "".join("\\x%2x" % ord(char) for char in arg)
 
@@ -54,20 +48,7 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
         fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
-        if smtpport is None:
-            smtphostport = smtphost
-        else:
-            smtphostport = "%s/%d" % (smtphost, smtpport)
-        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
-        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
-        if lmtpport is None:
-            lmtphostport = lmtphost
-        else:
-            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
@@ -79,7 +60,7 @@ def run(debug):
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport if fetch['scan'] else lmtphostport,
+                smtphost=f'{os.environ["SMTP_ADDRESS"]}' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}/2525',
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
                 options=options,
@@ -118,14 +99,15 @@ if __name__ == "__main__":
     os.chmod("/data/fetchids", 0o700)
     os.setgid(id_fetchmail.pw_gid)
     os.setuid(id_fetchmail.pw_uid)
+    config = system.set_env()
     while True:
-        delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
+        delay = int(os.environ.get('FETCHMAIL_DELAY', 60))
         print("Sleeping for {} seconds".format(delay))
         time.sleep(delay)
 
-        if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'):
+        if not config.get('FETCHMAIL_ENABLED', True):
             print("Fetchmail disabled, skipping...")
             continue
 
-        run(os.environ.get("DEBUG", None) == "True")
+        run(config.get('DEBUG', False))
         sys.stdout.flush()

optional/unbound/start.py

@@ -3,9 +3,10 @@
 import os
 import logging as log
 import sys
-from socrate import conf
+from socrate import conf, system
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")
 

setup/flavors/compose/docker-compose.yml

@@ -113,6 +113,9 @@ services:
       - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
     depends_on:
       - front
+    {% if antivirus_enabled %}
+      - antivirus
+    {% endif %}
     {% if resolver_enabled %}
       - resolver
     dns:

towncrier/newsfragments/1341.misc

@@ -0,0 +1,4 @@
+Remove HOST_* variables, use *_ADDRESS everywhere instead. Please note that those should only contain a FQDN (no port number).
+Derive a different key for admin/SECRET_KEY; this will invalidate existing sessions
+Ensure that rspamd starts after clamav
+Only display a single HOSTNAME on the client configuration page

webmails/start.py

@@ -13,14 +13,13 @@ from socrate import conf, system
 env = os.environ
 
 logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
+system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context
 context = {}
 context.update(env)
 
 context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576))
-context["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-context["IMAP_ADDRESS"] = system.get_host_address_from_environment("IMAP", "imap")
 
 db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite")
 if db_flavor == "sqlite":
@@ -43,17 +42,6 @@ else:
     print(f"Unknown ROUNDCUBE_DB_FLAVOR: {db_flavor}", file=sys.stderr)
     exit(1)
 
-# derive roundcube secret key
-secret_key = env.get("SECRET_KEY")
-if not secret_key:
-    try:
-        secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip()
-    except Exception as exc:
-        print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr)
-        exit(2)
-
-context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
-context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
 conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules")
 
 # roundcube plugins
@@ -127,8 +115,7 @@ conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf"
 if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")
 
-# clean env
-[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")]
+system.clean_env()
 
 # run nginx
 os.system("php-fpm81")