Merge #2790

2790: Implement managesieve support r=mergify[bot] a=nextgens

## What type of PR?

Feature

## What does this PR do?

This is a better a alternative to #2773

Expose managesieve to the outside world.

### Related issue(s)
- close #2773
- #428
- #113
- #81
- #1222

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [x] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>

README.md

@@ -20,7 +20,7 @@ Main features include:
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
 - **Advanced email features**, aliases, domain aliases, custom routing
 - **Web access**, multiple Webmails and administration interface
-- **User features**, aliases, auto-reply, auto-forward, fetched accounts
+- **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
 - **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/), block malicious attachments
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing

core/admin/mailu/internal/nginx.py

@@ -13,7 +13,8 @@ STATUSES = {
     "authentication": ("Authentication credentials invalid", {
         "imap": "AUTHENTICATIONFAILED",
         "smtp": "535 5.7.8",
-        "pop3": "-ERR Authentication failed"
+        "pop3": "-ERR Authentication failed",
+        "sieve": "AuthFailed"
     }),
     "encryption": ("Must issue a STARTTLS command first", {
         "smtp": "530 5.7.0"
@@ -25,7 +26,7 @@ STATUSES = {
     }),
 }
 
-WEBMAIL_PORTS = ['10143', '10025']
+WEBMAIL_PORTS = ['14190', '10143', '10025']
 
 def check_credentials(user, password, ip, protocol=None, auth_port=None):
     if not user or not user.enabled or (protocol == "imap" and not user.enable_imap and not auth_port in WEBMAIL_PORTS) or (protocol == "pop3" and not user.enable_pop):
@@ -49,8 +50,8 @@ def handle_authentication(headers):
     """ Handle an HTTP nginx authentication request
     See: http://nginx.org/en/docs/mail/ngx_mail_auth_http_module.html#protocol
     """
-    method = headers["Auth-Method"]
+    method = headers["Auth-Method"].lower()
-    protocol = headers["Auth-Protocol"]
+    protocol = headers["Auth-Protocol"].lower()
     # Incoming mail, no authentication
     if method == "none" and protocol == "smtp":
         server, port = get_server(protocol, False)
@@ -120,7 +121,7 @@ def handle_authentication(headers):
             "Auth-Wait": 0
         }
     # Unexpected
-    return {}
+    raise Exception("SHOULD NOT HAPPEN")
 
 
 def get_status(protocol, status):
@@ -139,6 +140,8 @@ def get_server(protocol, authenticated=False):
             hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
             hostname, port = app.config['SMTP_ADDRESS'], 25
+    elif protocol == "sieve":
+        hostname, port = app.config['IMAP_ADDRESS'], 4190
     try:
         # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)

core/dovecot/conf/dovecot.conf

@@ -12,6 +12,7 @@ default_login_user = mail
 default_internal_group = dovecot
 
 haproxy_trusted_networks = {{ SUBNET }} {{ SUBNET6 }}
+login_trusted_networks = {{ SUBNET }} {{ SUBNET6 }}
 
 ###############
 # Mailboxes
@@ -142,7 +143,6 @@ service lmtp {
 service managesieve-login {
   inet_listener sieve {
     port = 4190
-    haproxy = yes
   }
 }
 

core/nginx/Dockerfile

@@ -17,18 +17,18 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-stream nginx-mod-mail openssl \
+  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin
-  ; rm /etc/nginx/conf.d/stream.conf
 
 COPY conf/ /conf/
 COPY --from=static /static/ /static/
 COPY *.py /
+COPY dovecot/proxy.conf dovecot/login.lua /dovecot_conf/
 
 RUN echo $VERSION >/version
 
-EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp
+EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 4190/tcp
 # EXPOSE 10025/tcp 10143/tcp 14190/tcp
-HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://127.0.0.1:10204/health
+HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://127.0.0.1:10204/health && kill -0 `cat /run/dovecot/master.pid`
 
 VOLUME ["/certs", "/overrides"]
 

core/nginx/certwatcher.py

@@ -27,6 +27,9 @@ class ChangeHandler(FileSystemEventHandler):
         if exists("/var/run/nginx.pid"):
             print("Reloading a running nginx")
             system("nginx -s reload")
+        if os.path.exists("/run/dovecot/master.pid"):
+            print("Reloading a running dovecot")
+            os.system("doveadm reload")
 
     @staticmethod
     def reexec_config():

core/nginx/conf/nginx.conf

@@ -5,7 +5,6 @@ pcre_jit on;
 error_log /dev/stderr notice;
 pid /var/run/nginx.pid;
 load_module "modules/ngx_mail_module.so";
-load_module "modules/ngx_stream_module.so";
 
 events {
     worker_connections 1024;
@@ -74,6 +73,9 @@ http {
         return 301 https://$host$request_uri;
       }
 
+      location /health {
+        return 204;
+      }
     }
     {% endif %}
 
@@ -273,7 +275,6 @@ http {
         proxy_pass_request_body off;
         proxy_set_header Content-Length "";
       }
-
       location /health {
         return 204;
       }
@@ -302,25 +303,6 @@ http {
     include /etc/nginx/conf.d/*.conf;
 }
 
-stream {
-    log_format main '$remote_addr [$time_local] '
-    		'$protocol $status $bytes_sent $bytes_received '
-    		'$session_time "$upstream_addr" '
-    		'"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
-    access_log /dev/stdout main;
-
-    # managesieve
-    server {
-      listen 14190;
-      resolver {{ RESOLVER }} valid=30s;
-
-      proxy_connect_timeout 1s;
-      proxy_timeout 1m;
-      proxy_protocol on;
-      proxy_pass {{ IMAP_ADDRESS }}:4190;
-    }
-}
-
 mail {
     server_name {{ HOSTNAMES.split(",")[0] }};
     auth_http http://127.0.0.1:8000/auth/email;

core/nginx/config.py

@@ -55,3 +55,7 @@ conf.jinja("/conf/proxy.conf", args, "/etc/nginx/proxy.conf")
 conf.jinja("/conf/nginx.conf", args, "/etc/nginx/nginx.conf")
 if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")
+conf.jinja("/dovecot_conf/login.lua", args, "/etc/dovecot/login.lua")
+conf.jinja("/dovecot_conf/proxy.conf", args, "/etc/dovecot/proxy.conf")
+if os.path.exists("/run/dovecot/master.pid"):
+    os.system("doveadm reload")

core/nginx/dovecot/login.lua

@@ -0,0 +1,41 @@
+function script_init()
+  return 0
+end
+
+function script_deinit()
+end
+
+local http_client = dovecot.http.client {
+    timeout = 2000;
+    max_attempts = 3;
+}
+
+function auth_passdb_lookup(req)
+  local auth_request = http_client:request {
+    url = "http://{{ ADMIN_ADDRESS }}/internal/auth/email";
+  }
+  auth_request:add_header('Auth-Port', req.local_port)
+  auth_request:add_header('Auth-User', req.user)
+  auth_request:add_header('Auth-Pass', req.password)
+  auth_request:add_header('Auth-Protocol', req.service)
+  auth_request:add_header('Client-IP', req.remote_ip)
+  auth_request:add_header('Client-Port', req.remote_port)
+  auth_request:add_header('Auth-SSL', req.secured)
+  auth_request:add_header('Auth-Method', req.mechanism)
+  local auth_response = auth_request:submit()
+  local resp_status = auth_response:status()
+
+  if resp_status == 200
+  then
+    if auth_response:header('Auth-Status') == 'OK'
+    then
+      local server = auth_response:header('Auth-Server')
+      local port = auth_response:header('Auth-Port')
+      return dovecot.auth.PASSDB_RESULT_OK, "proxy=y host=" .. server .. " port=" .. port .. " nopassword=Y"
+    else
+      return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
+    end
+  else
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, ""
+  end
+end

core/nginx/dovecot/proxy.conf

@@ -0,0 +1,67 @@
+###############
+# General
+###############
+log_path = /dev/stderr
+auth_verbose=yes
+mail_debug=yes
+login_log_format_elements = user=<%u> method=%m rip=%r rport=%b lip=%l lport=%a mpid=%e %c
+protocols = sieve
+postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
+hostname = {{ HOSTNAMES.split(",")[0] }}
+submission_host = {{ FRONT_ADDRESS }}
+
+default_internal_user = dovecot
+default_login_user = mail
+default_internal_group = dovecot
+
+haproxy_trusted_networks = {% if REAL_IP_FROM %}{% for from_ip in REAL_IP_FROM.split(',') %}{{ from_ip }} {% endfor %}{% endif %}
+
+###############
+# Authentication
+###############
+auth_username_chars =
+auth_mechanisms = plain login
+
+{%- if TLS %}
+ssl = required
+ssl_cert = <{{ TLS[0] }}
+ssl_key = <{{ TLS[1] }}
+{%- if TLS_FLAVOR in ['letsencrypt','mail-letsencrypt'] %}
+ssl_alt_cert = <{{ TLS[2] }}
+ssl_alt_key = <{{ TLS[3] }}
+{% endif %}
+# intermediate configuration
+ssl_min_protocol = TLSv1.2
+ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ssl_prefer_server_ciphers = no
+ssl_dh = </conf/dhparam.pem
+ssl_options = no_compression no_ticket
+{% else %}
+disable_plaintext_auth = no
+protocol sieve {
+  ssl = no
+}
+{% endif %}
+
+passdb {
+  driver = lua
+  args = file=/etc/dovecot/login.lua blocking=yes
+}
+
+service auth-worker {
+  user = dovenull
+  group = dovenull
+}
+
+service managesieve-login {
+  executable = managesieve-login
+  inet_listener sieve {
+    port = 4190
+{%- if PROXY_PROTOCOL in ['all', 'mail'] %}
+    haproxy = yes
+{% endif %}
+  }
+  inet_listener sieve-webmail {
+    port = 14190
+  }
+}

core/nginx/start.py

@@ -16,4 +16,5 @@ elif os.environ["TLS_FLAVOR"] in [ "mail", "cert" ]:
     subprocess.Popen(["/certwatcher.py"])
 
 subprocess.call(["/config.py"])
+os.system("dovecot -c /etc/dovecot/proxy.conf")
 os.execv("/usr/sbin/nginx", ["nginx", "-g", "daemon off;"])

docs/compose/requirements.rst

@@ -63,7 +63,7 @@ address for your mail server and that you have a dedicated hostname
 with forward and reverse DNS entries for this IP address.
 
 Also, your host must not listen on ports ``25``, ``80``, ``110``, ``143``,
-``443``, ``465``, ``587``, ``993`` or ``995`` as these are used by Mailu
+``443``, ``465``, ``587``, ``993``, ``995`` nor ``4190`` as these are used by Mailu
 services. Therefore, you should disable or uninstall any program that is
 listening on these ports (or have them listen on a different port). For
 instance, on a default Debian install:

docs/index.rst

@@ -26,7 +26,7 @@ Main features include:
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
 - **Advanced email features**, aliases, domain aliases, custom routing
 - **Web access**, multiple Webmails and administration interface
-- **User features**, aliases, auto-reply, auto-forward, fetched accounts
+- **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
 - **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, `Snuffleupagus <https://github.com/jvoisin/snuffleupagus/>`_, block malicious attachments
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing

setup/flavors/compose/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       options:
         tag: mailu-front
     ports:
-    {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
+    {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993, 4190) %}
     {% if bind4 %}
       - "{{ bind4 }}:{{ port }}:{{ port }}"
     {% endif %}

tests/compose/core/05_connectivity.py

@@ -3,7 +3,8 @@
 import imaplib
 import poplib
 import smtplib
-import os
+import sys
+import managesieve
 
 SERVER='localhost'
 USERNAME='user@mailu.io'
@@ -26,7 +27,7 @@ def test_imap(server, username, password):
         with imaplib.IMAP4(server) as conn:
             conn.login(username, password)
         print(f'Authenticating to imap://{username}:{password}@{server}:143/ worked without STARTTLS!')
-        os.exit(102)
+        sys.exit(102)
     except imaplib.IMAP4.error:
         print('NOK - expected')
 
@@ -54,7 +55,7 @@ def test_pop3(server, username, password):
         conn.pass_(password)
         conn.close()
         print(f'Authenticating to pop3://{username}:{password}@{server}:110/ worked without STARTTLS!')
-        os.exit(103)
+        sys.exit(103)
     except poplib.error_proto:
         print('NOK - expected')
 
@@ -77,7 +78,7 @@ def test_SMTP(server, username, password):
             conn.ehlo()
             conn.login(username, password)
             print(f'Authenticating to smtp://{username}:{password}@{server}:587/ worked!')
-            os.exit(104)
+            sys.exit(104)
     except smtplib.SMTPNotSupportedError:
         print('NOK - expected')
     #port 25 should fail
@@ -89,7 +90,7 @@ def test_SMTP(server, username, password):
             conn.ehlo()
             conn.login(username, password)
             print(f'Authenticating to smtps://{username}:{password}@{server}:25/ worked!')
-            os.exit(105)
+            sys.exit(105)
     except smtplib.SMTPNotSupportedError:
         print('NOK - expected')
     try:
@@ -98,11 +99,36 @@ def test_SMTP(server, username, password):
             conn.ehlo()
             conn.login(username, password)
             print(f'Authenticating to smtp://{username}:{password}@{server}:25/ worked without STARTTLS!')
-            os.exit(106)
+            sys.exit(106)
     except smtplib.SMTPNotSupportedError:
         print('NOK - expected')
 
+def test_managesieve(server, username, password):
+    print(f'Authenticating to sieve://{username}:{password}@{server}:4190/')
+    m=managesieve.MANAGESIEVE(server)
+    try:
+        m.login('PLAIN', username, password)
+        print(f'Worked without STARTTLS!')
+        sys.exit(107)
+    except managesieve.MANAGESIEVE.abort:
+        pass
+
+    m=managesieve.MANAGESIEVE(server, use_tls=True)
+    if m.login('', username, 'wrongpass') != 'NO':
+        print(f'Authenticating to sieve://{username}:{password}@{server}:4190/ with wrong creds has worked!')
+        sys.exit(108)
+
+    if m.login('', username, password) != 'OK':
+        print(f'Authenticating to sieve://{username}:{password}@{server}:4190/ has failed!')
+        sys.exit(109)
+    
+    if m.listscripts()[0] != 'OK':
+        print(f'Listing scripts failed!')
+        sys.exit(110)
+    print('OK')
+
 if __name__ == '__main__':
     test_imap(SERVER, USERNAME, PASSWORD)
     test_pop3(SERVER, USERNAME, PASSWORD)
     test_SMTP(SERVER, USERNAME, PASSWORD)
+    test_managesieve(SERVER, USERNAME, PASSWORD)

tests/compose/core/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       - "127.0.0.1:995:995"
       - "127.0.0.1:143:143"
       - "127.0.0.1:993:993"
+      - "127.0.0.1:4190:4190"
     volumes:
       - "/mailu/certs:/certs"
 

tests/requirements.txt

@@ -1,2 +1,3 @@
 docker==4.2.2
 colorama==0.4.3
+managesieve==0.7.1

towncrier/newsfragments/81.feature

@@ -0,0 +1 @@
+Add support for managesieve

webmails/roundcube/config/config.inc.php

@@ -25,7 +25,14 @@ $config['smtp_user'] = '%u';
 $config['smtp_pass'] = '%p';
 
 // Sieve script management
-$config['managesieve_host'] = '{{ FRONT_ADDRESS or "front" }}:14190';
+$config['managesieve_host'] = 'tls://{{ FRONT_ADDRESS or "front" }}:14190';
+$config['managesieve_conn_options'] = array(
+  'ssl'         => array(
+     'verify_peer'  => false,
+     'verify_peer_name' => false,
+     'allow_self_signed' => true,
+   ),
+);
 $config['managesieve_mbox_encoding'] = 'UTF8';
 
 // roundcube customization

webmails/snappymail/defaults/default.json

@@ -34,12 +34,12 @@
     "Sieve": {
         "host": "{{ FRONT_ADDRESS }}",
         "port": 14190,
-        "secure": 0,
+        "type": 2,
         "shortLogin": false,
         "ssl": {
             "verify_peer": false,
             "verify_peer_name": false,
-            "allow_self_signed": false,
+            "allow_self_signed": true,
             "SNI_enabled": true,
             "disable_compression": true,
             "security_level": 1