Merge #2529

2529: Improve fetchmail r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

Improve fetchmail:
- allow delivery via LMTP (faster, bypassing the filters)
- allow several folders to be retrieved
- run fetchmail as non-root
- tweak the compose file to ensure we have all the dependencies

### Related issue(s)
- closes #1231 
- closes #2246 
- closes #711

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [ ] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.

Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>
Co-authored-by: Florent Daigniere <nextgens@users.noreply.github.com>

optional/fetchmail/fetchmail.py

@@ -2,11 +2,14 @@
 
 import time
 import os
+from pathlib import Path
+from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
 import re
 import requests
+from socrate import system
 import sys
 import traceback
 
@@ -14,6 +17,7 @@ import traceback
 FETCHMAIL = """
 fetchmail -N \
     --idfile /data/fetchids --uidl \
+    --pidfile /dev/shm/fetchmail.pid \
     --sslcertck --sslcertpath /etc/ssl/certs \
     -f {}
 """
@@ -24,7 +28,9 @@ poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
     is "{user_email}"
     smtphost "{smtphost}"
+    {folders}
     {options}
+    {lmtp}
 """
 
 
@@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
+        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
+        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+        fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
+        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
         if smtpport is None:
             smtphostport = smtphost
         else:
             smtphostport = "%s/%d" % (smtphost, smtpport)
+        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
+        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
+        if lmtpport is None:
+            lmtphostport = lmtphost
+        else:
+            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
             options += " ssl" if fetch["tls"] else ""
             options += " keep" if fetch["keep"] else " fetchall"
+            folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
             fetchmailrc += RC_LINE.format(
                 user_email=escape_rc_string(fetch["user_email"]),
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport,
+                smtphost=smtphostport if fetch['scan'] else lmtphostport,
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
-                options=options
+                options=options,
+                folders=folders,
+                lmtp='' if fetch['scan'] else 'lmtp',
             )
             if debug:
                 print(fetchmailrc)
@@ -86,14 +103,21 @@ def run(debug):
                         user_info in error_message):
                     print(error_message)
             finally:
-                requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
-                    json=error_message.split("\n")[0]
+                requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
+                    json=error_message.split('\n')[0]
                 )
     except Exception:
         traceback.print_exc()
 
 
 if __name__ == "__main__":
+    id_fetchmail = getpwnam('fetchmail')
+    Path('/data/fetchids').touch()
+    os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chmod("/data/fetchids", 0o700)
+    os.setgid(id_fetchmail.pw_gid)
+    os.setuid(id_fetchmail.pw_uid)
     while True:
         delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
         print("Sleeping for {} seconds".format(delay))