Merge #3412

3412: misc fixes (backport #3407) r=mergify[bot] a=mergify[bot]

## What type of PR?

bug-fix

## What does this PR do?

Update the documentation: clarify that dovecot also needs to be reloaded if custom certs are in use
Fix a 'fatal error' that may be encountered when using snappymail
Fix ``INBOUND_TLS_ENFORCE`` (something you should never use)
Fix ``DEFAULT_QUOTA``
Increase the size of php/nginx buffers on webmail
Maybe fix utf-8 decoding problems in socrate
Maybe fix utf-8 problems in fetchmail folder names

### Related issue(s)
- closes #3401 
- closes #3405
- closes #3403 
- closes #3379
- closes #3272
- closes #2996
- #3398

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [x] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.
<hr>This is an automatic backport of pull request #3407 done by [Mergify](https://mergify.com).

Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>
Co-authored-by: Florent Daigniere <nextgens@users.noreply.github.com>

core/admin/mailu/internal/nginx.py

@@ -18,7 +18,11 @@ STATUSES = {
         "sieve": "AuthFailed"
     }),
     "encryption": ("Must issue a STARTTLS command first", {
-        "smtp": "530 5.7.0"
+        "imap": "PRIVACYREQUIRED",
+        "smtp": "530 5.7.0",
+        "submission": "530 5.7.0",
+        "pop3": "-ERR Authentication canceled.",
+        "sieve": "ENCRYPT-NEEDED"
     }),
     "ratelimit": ("Temporary authentication failure (rate-limit)", {
         "imap": "LIMIT",
@@ -68,7 +72,7 @@ def handle_authentication(headers):
     # Incoming mail, no authentication
     if method in ['', 'none'] and protocol in ['smtp', 'lmtp']:
         server, port = get_server(protocol, False)
-        if app.config["INBOUND_TLS_ENFORCE"]:
+        if app.config["INBOUND_TLS_ENFORCE"] and protocol == 'smtp':
             if "Auth-SSL" in headers and headers["Auth-SSL"] == "on":
                 return {
                     "Auth-Status": "OK",

core/admin/mailu/ui/views/users.py

@@ -24,7 +24,7 @@ def user_create(domain_name):
             flask.url_for('.user_list', domain_name=domain.name))
     form = forms.UserForm()
     form.pw.validators = [wtforms.validators.DataRequired()]
-    form.quota_bytes.default = app.config['DEFAULT_QUOTA']
+    form.quota_bytes.default = int(app.config['DEFAULT_QUOTA'])
     if domain.max_quota_bytes:
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=domain.max_quota_bytes)]

core/base/libs/socrate/socrate/system.py

@@ -158,7 +158,7 @@ def forward_text_lines(src, dst):
 
 # runs a process and passes its standard/error output to the standard/error output of the current python script
 def run_process_and_forward_output(cmd):
-    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
 
     stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout))
     stdout_thread.daemon = True

docs/maintain.rst

@@ -47,6 +47,7 @@ In the case of *certbot* you could write a script to be executed as `deploy hook
   cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem || exit 1
   cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem || exit 1
   docker exec mailu_front_1 nginx -s reload
+  docker exec mailu_front_1 doveadm reload
 
 And the certbot command you will use in crontab would look something like:
 

optional/fetchmail/fetchmail.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import binascii
 import time
 import os
 from pathlib import Path
@@ -32,6 +33,19 @@ poll "{host}" proto {protocol}  port {port}
     {lmtp}
 """
 
+def imaputf7encode(s):
+    """Encode a string into RFC2060 aka IMAP UTF7"""
+    out = ''
+    enc = ''
+    for c in s.replace('&','&-') + 'X':
+        if '\x20' <= c <= '\x7f':
+            if enc:
+                out += f'&{binascii.b2a_base64(enc.encode("utf-16-be")).rstrip(b"\n=").replace(b"/", b",").decode("ascii")}-'
+                enc = ''
+            out += c
+        else:
+            enc += c
+    return out[:-1]
 
 def escape_rc_string(arg):
     return "".join("\\x%2x" % ord(char) for char in arg)
@@ -54,7 +68,7 @@ def run(debug):
             options = "options antispam 501, 504, 550, 553, 554"
             options += " ssl" if fetch["tls"] else ""
             options += " keep" if fetch["keep"] else " fetchall"
-            folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
+            folders = f"folders {",".join(f'"{imaputf7encode(item).replace('"',r"\34")}"' for item in fetch["folders"]) or '"INBOX"'}"
             fetchmailrc += RC_LINE.format(
                 user_email=escape_rc_string(fetch["user_email"]),
                 protocol=fetch["protocol"],

towncrier/newsfragments/2296.bugfix

@@ -0,0 +1 @@
+Ensure fetchmail can deal with special characters in folder names

towncrier/newsfragments/3272.bugfix

@@ -0,0 +1 @@
+Increase the size of buffers for webmail

towncrier/newsfragments/3379.bugfix

@@ -0,0 +1 @@
+Fix #3379: DEFAULT_QUOTA

towncrier/newsfragments/3401.bugfix

@@ -0,0 +1 @@
+Fix an error that can occur when using snappymail

towncrier/newsfragments/3403.bugfix

@@ -0,0 +1 @@
+fix INBOUND_TLS_ENFORCE

towncrier/newsfragments/3405.bugfix

@@ -0,0 +1 @@
+Update the documentation: ensure that users reload dovecot too if they manually configure certificates

webmails/nginx-webmail.conf

@@ -55,6 +55,16 @@ server {
         {% else %}
         fastcgi_param SCRIPT_NAME {{WEB_WEBMAIL}}/$fastcgi_script_name;
         {% endif %}
+
+        # fastcgi buffers for php-fpm #
+        fastcgi_buffers 16 32k;
+        fastcgi_buffer_size 64k;
+        fastcgi_busy_buffers_size 64k;
+
+        # nginx buffers #
+        proxy_buffer_size 128k;
+        proxy_buffers 4 256k;
+        proxy_busy_buffers_size 256k;
     }
 
     location ~ (^|/)\. {

webmails/snuffleupagus.rules

@@ -130,6 +130,7 @@ sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ht").
 
 # Logging lockdown
 sp.disable_function.function("ini_set").param("option").value_r("error_log").drop()
+sp.disable_function.function("ini_set").param("option").value_r("display_errors").filename_r("/var/www/snappymail/snappymail/v/[0-9]+\.[0-9]+\.[0-9]+/app/libraries/snappymail/shutdown.php").allow();
 sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop()
 
 sp.auto_cookie_secure.enable();