diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index e05c4f18..1ced1a05 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -31,30 +31,29 @@ def _coerce_value(value): class LogFilter(object): def __init__(self, stream, re_patterns): - self.stream = stream - if isinstance(re_patterns, list): - self.pattern = re.compile('|'.join([fr'(?:{pattern})' for pattern in re_patterns])) - elif isinstance(re_patterns, str): - self.pattern = re.compile(re_patterns) - else: - self.pattern = re_patterns - self.found = False + self.stream = stream + self.pattern = re.compile(b'|'.join([b''.join([b'(?:', pattern, b')']) for pattern in re_patterns])) + self.buffer = b'' def __getattr__(self, attr_name): return getattr(self.stream, attr_name) def write(self, data): - if data == '\n' and self.found: - self.found = False - else: - if not self.pattern.search(data): - self.stream.write(data) + if type(data) is str: + data = data.encode('utf-8') + self.buffer += data + while b'\n' in self.buffer: + line, cr, rest = self.buffer.partition(b'\n') + if not self.pattern.search(line): + self.stream.buffer.write(line) + self.stream.buffer.write(cr) self.stream.flush() - else: - # caught bad pattern - self.found = True + self.buffer = rest def flush(self): + # write out buffer on flush even if it's not a complete line + if self.buffer and not self.pattern.search(self.buffer): + self.stream.buffer.write(self.buffer) self.stream.flush() def _is_compatible_with_hardened_malloc(): @@ -109,7 +108,7 @@ def set_env(required_secrets=[], log_filters=[]): for secret in required_secrets: os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest() - os.system('find /run -xdev -type f -name \*.pid -print -delete') + os.system(r'find /run -xdev -type f -name \*.pid -print -delete') return { key: _coerce_value(os.environ.get(key, value)) @@ -169,7 +168,7 @@ def forward_text_lines(src, dst): # runs a process and passes its standard/error output to the standard/error output of the current python script def run_process_and_forward_output(cmd): - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout)) stdout_thread.daemon = True @@ -179,4 +178,7 @@ def run_process_and_forward_output(cmd): stderr_thread.daemon = True stderr_thread.start() - process.wait() + rc = process.wait() + sys.stdout.flush() + sys.stderr.flush() + return rc diff --git a/core/dovecot/start.py b/core/dovecot/start.py index b162db95..83afa1de 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -7,7 +7,9 @@ import multiprocessing from podop import run_server from socrate import system, conf -system.set_env(log_filters=[r'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$']) +system.set_env(log_filters=[ + rb'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$' +]) def start_podop(): system.drop_privs_to('mail') diff --git a/core/nginx/start.py b/core/nginx/start.py index a50abec2..48f62511 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -4,7 +4,9 @@ import os import subprocess from socrate import system -system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$') +system.set_env(log_filters=[ + rb'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$' +]) # Check if a stale pid file exists if os.path.exists("/var/run/nginx.pid"): diff --git a/core/postfix/start.py b/core/postfix/start.py index 7daf7c3b..aaa141a8 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -11,10 +11,10 @@ from podop import run_server from socrate import system, conf system.set_env(log_filters=[ - r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', - r'haproxy read\: short protocol header\: QUIT$', - r'discarding EHLO keywords\: PIPELINING$' - ]) + rb'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', + rb'haproxy read\: short protocol header\: QUIT$', + rb'discarding EHLO keywords\: PIPELINING$' +]) os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid") diff --git a/towncrier/newsfragments/3618.bugfix b/towncrier/newsfragments/3618.bugfix new file mode 100644 index 00000000..fb72e3a2 --- /dev/null +++ b/towncrier/newsfragments/3618.bugfix @@ -0,0 +1 @@ +Filter logs line based and in binary mode without decoding utf-8