Merge #2847

2847: Use dovecot-proxy r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

Use dovecot-proxy instead of nginx's mail module.

This will be required if we want to have BURL, XOAUTH2 or HA/load-balancing (with more than one IMAP instance).

pros:
- fixes $remote_port

cons:
- seems to break SSL resumption

### Related issue(s)
- closes #2848

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [ ] In case of feature or enhancement: documentation updated accordingly
- [ ] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>

core/admin/mailu/internal/nginx.py

@@ -13,6 +13,7 @@ STATUSES = {
     "authentication": ("Authentication credentials invalid", {
         "imap": "AUTHENTICATIONFAILED",
         "smtp": "535 5.7.8",
+        "submission": "535 5.7.8",
         "pop3": "-ERR Authentication failed",
         "sieve": "AuthFailed"
     }),
@@ -56,7 +57,7 @@ def handle_authentication(headers):
     method = headers["Auth-Method"].lower()
     protocol = headers["Auth-Protocol"].lower()
     # Incoming mail, no authentication
-    if method == "none" and protocol == "smtp":
+    if method in ['', 'none'] and protocol in ['smtp', 'lmtp']:
         server, port = get_server(protocol, False)
         if app.config["INBOUND_TLS_ENFORCE"]:
             if "Auth-SSL" in headers and headers["Auth-SSL"] == "on":
@@ -79,7 +80,7 @@ def handle_authentication(headers):
                 "Auth-Port": port
             }
     # Authenticated user
-    elif method == "plain":
+    elif method in ['plain', 'login']:
         is_valid_user = False
         # According to RFC2616 section 3.7.1 and PEP 3333, HTTP headers should
         # be ASCII and are generally considered ISO8859-1. However when passing
@@ -122,7 +123,7 @@ def handle_authentication(headers):
             "Auth-Wait": 0
         }
     # Unexpected
-    raise Exception("SHOULD NOT HAPPEN")
+    raise Exception(f"SHOULD NOT HAPPEN {protocol} {method}")
 
 
 def get_status(protocol, status):
@@ -132,16 +133,20 @@ def get_status(protocol, status):
     return status, codes[protocol]
 
 def get_server(protocol, authenticated=False):
-    if protocol == "imap":
+    if protocol == 'imap':
         hostname, port = app.config['IMAP_ADDRESS'], 143
-    elif protocol == "pop3":
+    elif protocol == 'pop3':
         hostname, port = app.config['IMAP_ADDRESS'], 110
-    elif protocol == "smtp":
+    elif protocol == 'smtp':
         if authenticated:
             hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
             hostname, port = app.config['SMTP_ADDRESS'], 25
-    elif protocol == "sieve":
+    elif protocol == 'submission':
+        hostname, port = app.config['SMTP_ADDRESS'], 10025
+    elif protocol == 'lmtp':
+        hostname, port = app.config['IMAP_ADDRESS'], 2525
+    elif protocol == 'sieve':
         hostname, port = app.config['IMAP_ADDRESS'], 4190
     try:
         # test if hostname is already resolved to an ip address

core/admin/mailu/internal/views/auth.py

@@ -31,7 +31,7 @@ def nginx_authentication():
         if int(flask.request.headers['Auth-Login-Attempt']) < 10:
             response.headers['Auth-Wait'] = '3'
         return response
-    raw_password = urllib.parse.unquote(headers["Auth-Pass"])
+    raw_password = urllib.parse.unquote(headers['Auth-Pass']) if 'Auth-Pass' in headers else ''
     headers = nginx.handle_authentication(flask.request.headers)
     response = flask.Response()
     for key, value in headers.items():

core/admin/mailu/models.py

@@ -421,7 +421,7 @@ class Email(object):
         """ send an email to the address """
         try:
             f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-            with smtplib.LMTP(host=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
+            with smtplib.LMTP(host=app.config['FRONT_ADDRESS'], port=2525) as lmtp:
                 to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
                 msg = text.MIMEText(body)
                 msg['Subject'] = subject

core/dovecot/Dockerfile

@@ -15,7 +15,7 @@ COPY start.py /
 
 RUN echo $VERSION >/version
 
-EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
+# EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
 HEALTHCHECK CMD kill -0 `cat /run/dovecot/master.pid`
 
 VOLUME ["/mail"]

core/dovecot/conf/dovecot.conf

@@ -11,7 +11,6 @@ default_internal_user = dovecot
 default_login_user = mail
 default_internal_group = dovecot
 
-haproxy_trusted_networks = {{ SUBNET }} {{ SUBNET6 }}
 login_trusted_networks = {{ SUBNET }} {{ SUBNET6 }}
 
 ###############
@@ -112,14 +111,12 @@ protocol pop3 {
 service imap-login {
   inet_listener imap {
     port = 143
-    haproxy = yes
   }
 }
 
 service pop3-login {
   inet_listener pop3 {
     port = 110
-    haproxy = yes
   }
 }
 
@@ -191,6 +188,12 @@ plugin {
   imapsieve_mailbox2_before = file:/conf/report-ham.sieve
 }
 
+service anvil {
+  unix_listener anvil-auth-penalty {
+    mode = 0
+  }
+}
+
 ###############
 # Extensions
 ###############

core/nginx/Dockerfile

@@ -17,7 +17,7 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin
+  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin dovecot-lmtpd dovecot-pop3d dovecot-submissiond
 
 COPY conf/ /conf/
 COPY --from=static /static/ /static/

core/nginx/conf/nginx.conf

@@ -321,28 +321,6 @@ mail {
 
     # Advertise real capabilities of backends (postfix/dovecot)
     smtp_capabilities PIPELINING "SIZE {{ MESSAGE_SIZE_LIMIT }}" ETRN ENHANCEDSTATUSCODES 8BITMIME DSN;
-    pop3_capabilities TOP UIDL RESP-CODES PIPELINING AUTH-RESP-CODE USER;
-    imap_capabilities IMAP4 IMAP4rev1 UIDPLUS SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+;
-
-    # Default SMTP server for the webmail (no encryption, but authentication)
-    server {
-      listen 10025;
-      protocol smtp;
-      smtp_auth plain;
-      auth_http_header Auth-Port 10025;
-      auth_http_header Client-Port $remote_port;
-    }
-
-    # Default IMAP server for the webmail (no encryption, but authentication)
-    server {
-      listen 10143;
-      protocol imap;
-      smtp_auth plain;
-      auth_http_header Auth-Port 10143;
-      auth_http_header Client-Port $remote_port;
-      # ensure we talk HAPROXY protocol to the backends
-      proxy_protocol on;
-    }
 
     # SMTP is always enabled, to avoid losing emails when TLS is failing
     server {
@@ -367,92 +345,4 @@ mail {
       auth_http_header Auth-Port 25;
       auth_http_header Client-Port $remote_port;
     }
-
-    # All other protocols are disabled if TLS is failing
-    {% if not TLS_ERROR %}
-    server {
-      listen 143{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:143{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      {% if TLS %}
-      starttls only;
-      {% endif %}
-      protocol imap;
-      imap_auth plain;
-      auth_http_header Auth-Port 143;
-      auth_http_header Client-Port $remote_port;
-      # ensure we talk HAPROXY protocol to the backends
-      proxy_protocol on;
-    }
-
-    server {
-      listen 110{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:110{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      {% if TLS %}
-      starttls only;
-      {% endif %}
-      protocol pop3;
-      pop3_auth plain;
-      auth_http_header Auth-Port 110;
-      auth_http_header Client-Port $remote_port;
-      # ensure we talk HAPROXY protocol to the backends
-      proxy_protocol on;
-    }
-
-    server {
-      listen 587{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:587{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      {% if TLS %}
-      starttls only;
-      {% endif %}
-      protocol smtp;
-      smtp_auth plain login;
-      auth_http_header Auth-Port 587;
-      auth_http_header Client-Port $remote_port;
-    }
-
-    {% if TLS %}
-    server {
-      listen 465 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:465 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      protocol smtp;
-      smtp_auth plain login;
-      auth_http_header Auth-Port 465;
-      auth_http_header Client-Port $remote_port;
-    }
-
-    server {
-      listen 993 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:993 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      protocol imap;
-      imap_auth plain;
-      auth_http_header Auth-Port 993;
-      auth_http_header Client-Port $remote_port;
-      # ensure we talk HAPROXY protocol to the backends
-      proxy_protocol on;
-    }
-
-    server {
-      listen 995 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% if SUBNET6 %}
-      listen [::]:995 ssl{% if PROXY_PROTOCOL in ['all', 'mail'] %} proxy_protocol{% endif %};
-{% endif %}
-      protocol pop3;
-      pop3_auth plain;
-      auth_http_header Auth-Port 995;
-      auth_http_header Client-Port $remote_port;
-      # ensure we talk HAPROXY protocol to the backends
-      proxy_protocol on;
-    }
-    {% endif %}
-    {% endif %}
 }

core/nginx/dovecot/login.lua

@@ -16,7 +16,10 @@ function auth_passdb_lookup(req)
   }
   auth_request:add_header('Auth-Port', req.local_port)
   auth_request:add_header('Auth-User', req.user)
-  auth_request:add_header('Auth-Pass', req.password)
+  if req.password ~= nil
+  then
+    auth_request:add_header('Auth-Pass', req.password)
+  end
   auth_request:add_header('Auth-Protocol', req.service)
   auth_request:add_header('Client-IP', req.remote_ip)
   auth_request:add_header('Client-Port', req.remote_port)
@@ -31,7 +34,7 @@ function auth_passdb_lookup(req)
     then
       local server = auth_response:header('Auth-Server')
       local port = auth_response:header('Auth-Port')
-      return dovecot.auth.PASSDB_RESULT_OK, "proxy=y host=" .. server .. " port=" .. port .. " nopassword=Y"
+      return dovecot.auth.PASSDB_RESULT_OK, "proxy=y host=" .. server .. " port=" .. port .. " nopassword=Y proxy_noauth=Y"
     else
       return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
     end

core/nginx/dovecot/proxy.conf

@@ -5,10 +5,21 @@ log_path = /dev/stderr
 auth_verbose=yes
 mail_debug=yes
 login_log_format_elements = user=<%u> method=%m rip=%r rport=%b lip=%l lport=%a mpid=%e %c
-protocols = sieve
+protocols = sieve imap pop3 lmtp submission
 postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }}
+submission_host = {{ SMTP_ADDRESS }}
+submission_relay_host = {{ SMTP_ADDRESS }}
+submission_relay_port = 10025
+submission_relay_trusted = yes
+submission_relay_ssl = no
+submission_max_mail_size = {{ MESSAGE_SIZE_LIMIT }}
+submission_backend_capabilities = 8BITMIME DSN VRFY
+submission_client_workarounds = mailbox-for-path whitespace-before-path
+# disable BURL
+imap_urlauth_host=
+lmtp_proxy = yes
+lmtp_client_workarounds = whitespace-before-path mailbox-for-path
 
 default_internal_user = dovecot
 default_login_user = mail
@@ -32,15 +43,12 @@ ssl_alt_key = <{{ TLS[3] }}
 {% endif %}
 # intermediate configuration
 ssl_min_protocol = TLSv1.2
-ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305
 ssl_prefer_server_ciphers = no
-ssl_dh = </conf/dhparam.pem
 ssl_options = no_compression no_ticket
 {% else %}
 disable_plaintext_auth = no
-protocol sieve {
-  ssl = no
-}
+ssl = no
 {% endif %}
 
 passdb {
@@ -51,6 +59,14 @@ passdb {
 service auth-worker {
   user = dovenull
   group = dovenull
+  unix_listener auth-worker {
+  }
+}
+
+service anvil {
+  unix_listener anvil-auth-penalty {
+    mode = 0
+  }
 }
 
 service managesieve-login {
@@ -65,3 +81,76 @@ service managesieve-login {
     port = 14190
   }
 }
+
+protocol imap {
+  mail_max_userip_connections = 20
+  imap_idle_notify_interval = 29mins
+}
+
+service imap-login {
+  inet_listener imap {
+    port = 143
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+  }
+  inet_listener imaps {
+    port = 993
+{%- if TLS %}
+    ssl = yes
+{% endif %}
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+  }
+  inet_listener imap-webmail {
+   port = 10143
+  }
+}
+
+service pop3-login {
+  inet_listener pop3 {
+    port = 110
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+  }
+  inet_listener pop3s {
+    port = 995
+{%- if TLS %}
+    ssl = yes
+{% endif %}
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+  }
+}
+
+recipient_delimiter = {{ RECIPIENT_DELIMITER }}
+service lmtp {
+  user = $default_internal_user
+  inet_listener lmtp {
+    port = 2525
+  }
+}
+
+service submission-login {
+   inet_listener submission {
+    port = 587
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+   }
+   inet_listener submissions {
+    port = 465
+{%- if TLS %}
+    ssl = yes
+{% endif %}
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+   }
+   inet_listener submission-webmail {
+    port = 10025
+   }
+}

core/postfix/conf/main.cf

@@ -81,7 +81,7 @@ virtual_mailbox_maps = ${podop}mailbox
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
 transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
-virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525
+virtual_transport = lmtp:inet:{{ FRONT_ADDRESS }}:2525
 
 # Sender and recipient canonical maps, mostly for SRS
 sender_canonical_maps = ${podop}sendermap

core/postfix/conf/master.cf

@@ -7,7 +7,7 @@ smtp      inet  n       -       n       -       1       smtpd
 # Internal SMTP service
 10025     inet  n       -       n       -       1       smtpd
   -o smtpd_sasl_auth_enable=yes
-  -o smtpd_discard_ehlo_keywords=pipelining
+  -o smtpd_discard_ehlo_keywords=pipelining,silent-discard
   -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
   -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %}
   -o cleanup_service_name=outclean

optional/fetchmail/fetchmail.py

@@ -60,7 +60,7 @@ def run(debug):
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=f'{os.environ["SMTP_ADDRESS"]}' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}/2525',
+                smtphost=f'{os.environ["FRONT_ADDRESS"]}' if fetch['scan'] else f'{os.environ["FRONT_ADDRESS"]}/2525',
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
                 options=options,

towncrier/newsfragments/2847.misc

@@ -0,0 +1 @@
+Switch from nginx mail module to dovecot-proxy. This will fix $remote_port and will enable us to work on new features such as BURL/XOAuth2/HA-load-balancing.

webmails/roundcube/config/config.inc.php

@@ -19,10 +19,24 @@ $config['request_path'] = '{{ WEB_WEBMAIL or "none" }}';
 $config['trusted_host_patterns'] = [ {{ HOSTNAMES.split(",") | map("tojson") | join(',') }}];
 
 // Mail servers
-$config['imap_host'] = '{{ FRONT_ADDRESS or "front" }}:10143';
-$config['smtp_host'] = '{{ FRONT_ADDRESS or "front" }}:10025';
+$config['imap_host'] = 'tls://{{ FRONT_ADDRESS or "front" }}:10143';
+$config['imap_conn_options'] = array(
+  'ssl'         => array(
+     'verify_peer'  => false,
+     'verify_peer_name' => false,
+     'allow_self_signed' => true,
+   ),
+);
+$config['smtp_host'] = 'tls://{{ FRONT_ADDRESS or "front" }}:10025';
 $config['smtp_user'] = '%u';
 $config['smtp_pass'] = '%p';
+$config['smtp_conn_options'] = array(
+  'ssl'         => array(
+     'verify_peer'  => false,
+     'verify_peer_name' => false,
+     'allow_self_signed' => true,
+   ),
+);
 
 // Sieve script management
 $config['managesieve_host'] = 'tls://{{ FRONT_ADDRESS or "front" }}:14190';

webmails/snappymail/defaults/default.json

@@ -3,7 +3,7 @@
     "IMAP": {
         "host": "{{ FRONT_ADDRESS }}",
         "port": 10143,
-        "secure": 0,
+        "secure": 2,
         "shortLogin": false,
         "ssl": {
             "verify_peer": false,
@@ -17,7 +17,7 @@
     "SMTP": {
         "host": "{{ FRONT_ADDRESS }}",
         "port": 10025,
-        "secure": 0,
+        "secure": 2,
         "shortLogin": false,
         "ssl": {
             "verify_peer": false,