Merge #3618

3618: Filter logs line based and in binary mode without decoding utf-8 r=mergify[bot] a=ghostwheel42

## What type of PR?

bug-fix

## What does this PR do?

try at fixing decoding errors when filtering logs

### Related issue(s)
- closes #3398



Co-authored-by: Alexander Graf <ghostwheel42@users.noreply.github.com>

core/base/libs/socrate/socrate/system.py

@@ -31,30 +31,29 @@ def _coerce_value(value):
 
 class LogFilter(object):
     def __init__(self, stream, re_patterns):
-        self.stream = stream
+        self.stream  = stream
-        if isinstance(re_patterns, list):
+        self.pattern = re.compile(b'|'.join([b''.join([b'(?:', pattern, b')']) for pattern in re_patterns]))
-            self.pattern = re.compile('|'.join([fr'(?:{pattern})' for pattern in re_patterns]))
+        self.buffer  = b''
-        elif isinstance(re_patterns, str):
-            self.pattern = re.compile(re_patterns)
-        else:
-            self.pattern = re_patterns
-        self.found = False
 
     def __getattr__(self, attr_name):
         return getattr(self.stream, attr_name)
 
     def write(self, data):
-        if data == '\n' and self.found:
+        if type(data) is str:
-            self.found = False
+            data = data.encode('utf-8')
-        else:
+        self.buffer += data
-            if not self.pattern.search(data):
+        while b'\n' in self.buffer:
-                self.stream.write(data)
+            line, cr, rest = self.buffer.partition(b'\n')
+            if not self.pattern.search(line):
+                self.stream.buffer.write(line)
+                self.stream.buffer.write(cr)
                 self.stream.flush()
-            else:
+            self.buffer = rest
-                # caught bad pattern
-                self.found = True
 
     def flush(self):
+        # write out buffer on flush even if it's not a complete line
+        if self.buffer and not self.pattern.search(self.buffer):
+            self.stream.buffer.write(self.buffer)
         self.stream.flush()
 
 def _is_compatible_with_hardened_malloc():
@@ -109,7 +108,7 @@ def set_env(required_secrets=[], log_filters=[]):
     for secret in required_secrets:
         os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
-    os.system('find /run -xdev -type f -name \*.pid -print -delete')
+    os.system(r'find /run -xdev -type f -name \*.pid -print -delete')
 
     return {
             key: _coerce_value(os.environ.get(key, value))
@@ -169,7 +168,7 @@ def forward_text_lines(src, dst):
 
 # runs a process and passes its standard/error output to the standard/error output of the current python script
 def run_process_and_forward_output(cmd):
-    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout))
     stdout_thread.daemon = True
@@ -179,4 +178,7 @@ def run_process_and_forward_output(cmd):
     stderr_thread.daemon = True
     stderr_thread.start()
 
-    process.wait()
+    rc = process.wait()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    return rc

core/dovecot/start.py

@@ -7,7 +7,9 @@ import multiprocessing
 from podop import run_server
 from socrate import system, conf
 
-system.set_env(log_filters=[r'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$'])
+system.set_env(log_filters=[
+    rb'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$'
+])
 
 def start_podop():
     system.drop_privs_to('mail')

core/nginx/start.py

@@ -4,7 +4,9 @@ import os
 import subprocess
 from socrate import system
 
-system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$')
+system.set_env(log_filters=[
+    rb'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$'
+])
 
 # Check if a stale pid file exists
 if os.path.exists("/var/run/nginx.pid"):

core/postfix/start.py

@@ -11,10 +11,10 @@ from podop import run_server
 from socrate import system, conf
 
 system.set_env(log_filters=[
-    r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
+    rb'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
-    r'haproxy read\: short protocol header\: QUIT$',
+    rb'haproxy read\: short protocol header\: QUIT$',
-    r'discarding EHLO keywords\: PIPELINING$'
+    rb'discarding EHLO keywords\: PIPELINING$'
-    ])
+])
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 

towncrier/newsfragments/3618.bugfix

@@ -0,0 +1 @@
+Filter logs line based and in binary mode without decoding utf-8