Fixed log filter not filtering out log messages for dovecot/nginx/postfix.

Fixed postfix not logging to standard out.
Fixed not all containers logging to journald.
Removed POSTFIX_LOG_FILE functionality. Added documentation on how to achieve the same (log to file) via journald & rsyslogd (see new FAQ entry 'How can I view and export the logs of a Mailu container?').

core/base/libs/socrate/socrate/system.py

@@ -6,6 +6,8 @@ import re
 from pwd import getpwnam
 import socket
 import tenacity
+import subprocess
+import threading
 
 @tenacity.retry(stop=tenacity.stop_after_attempt(100),
                 wait=tenacity.wait_random(min=2, max=5))
@@ -27,7 +29,7 @@ def _coerce_value(value):
     return value
 
 class LogFilter(object):
-    def __init__(self, stream, re_patterns, log_file):
+    def __init__(self, stream, re_patterns):
         self.stream = stream
         if isinstance(re_patterns, list):
             self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns]))
@@ -36,7 +38,6 @@ class LogFilter(object):
         else:
             self.pattern = re_patterns
         self.found = False
-        self.log_file = log_file
 
     def __getattr__(self, attr_name):
         return getattr(self.stream, attr_name)
@@ -48,12 +49,6 @@ class LogFilter(object):
             if not self.pattern.search(data):
                 self.stream.write(data)
                 self.stream.flush()
-                if self.log_file:
-                    try:
-                        with open(self.log_file, 'a', encoding='utf-8') as l:
-                            l.write(data)
-                    except:
-                        pass
             else:
                 # caught bad pattern
                 self.found = True
@@ -74,10 +69,10 @@ def _is_compatible_with_hardened_malloc():
                 return False
     return True
 
-def set_env(required_secrets=[], log_filters=[], log_file=None):
+def set_env(required_secrets=[], log_filters=[]):
     if log_filters:
-        sys.stdout = LogFilter(sys.stdout, log_filters, log_file)
-        sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
+        sys.stdout = LogFilter(sys.stdout, log_filters)
+        sys.stderr = LogFilter(sys.stderr, log_filters)
     log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
 
     if not 'LD_PRELOAD' in os.environ and _is_compatible_with_hardened_malloc():
@@ -115,3 +110,24 @@ def drop_privs_to(username='mailu'):
     os.setgid(pwnam.pw_gid)
     os.setuid(pwnam.pw_uid)
     os.environ['HOME'] = pwnam.pw_dir
+
+# forwards text lines from src to dst in an infinite loop
+def forward_text_lines(src, dst):
+    while True:
+        current_line = src.readline()
+        dst.write(current_line)
+
+
+# runs a process and passes its standard/error output to the standard/error output of the current python script
+def run_process_and_forward_output(cmd):
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout))
+    stdout_thread.daemon = True
+    stdout_thread.start()
+
+    stderr_thread = threading.Thread(target=forward_text_lines, args=(process.stderr, sys.stderr))
+    stderr_thread.daemon = True
+    stderr_thread.start()
+
+    process.wait()

core/dovecot/start.py

@@ -3,13 +3,11 @@
 import os
 import glob
 import multiprocessing
-import logging as log
-import sys
 
 from podop import run_server
 from socrate import system, conf
 
-system.set_env(log_filters=r'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$')
+system.set_env(log_filters=[r'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$'])
 
 def start_podop():
     system.drop_privs_to('mail')
@@ -35,4 +33,5 @@ os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
 
 multiprocessing.Process(target=start_podop).start()
-os.system("dovecot -c /etc/dovecot/dovecot.conf -F")
+cmd = ['/usr/sbin/dovecot', '-c', '/etc/dovecot/dovecot.conf', '-F']
+system.run_process_and_forward_output(cmd)

core/nginx/start.py

@@ -4,7 +4,7 @@ import os
 import subprocess
 from socrate import system
 
-system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$')
+system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$')
 
 # Check if a stale pid file exists
 if os.path.exists("/var/run/nginx.pid"):
@@ -17,4 +17,5 @@ elif os.environ["TLS_FLAVOR"] in [ "mail", "cert" ]:
 
 subprocess.call(["/config.py"])
 os.system("dovecot -c /etc/dovecot/proxy.conf")
-os.execv("/usr/sbin/nginx", ["nginx", "-g", "daemon off;"])
+cmd = ['/usr/sbin/nginx', '-g', 'daemon off;']
+system.run_process_and_forward_output(cmd)

core/postfix/start.py

@@ -13,8 +13,8 @@ from socrate import system, conf
 system.set_env(log_filters=[
     r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
     r'haproxy read\: short protocol header\: QUIT$',
-    r'discarding EHLO keywords\: PIPELINING$',
-    ], log_file=os.environ.get('POSTFIX_LOG_FILE'))
+    r'discarding EHLO keywords\: PIPELINING$'
+    ])
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -100,4 +100,5 @@ os.system("/usr/libexec/postfix/post-install meta_directory=/etc/postfix create-
 # Before starting postfix, we need to check permissions on /queue
 # in the event that postfix,postdrop id have changed
 os.system("postfix set-permissions")
-os.system("postfix start-fg")
+cmd = ['postfix', 'start-fg']
+system.run_process_and_forward_output(cmd)