tests: new sending and filtering functions (#3786)

* move log/filter functions into own file

* add ShellCheck global directives

* use new function for tracking logs

The new function, called `_send_email_with_mid`, aligns with suggestions
from @polarethene and is heavily simplified compared to its predecessor
`_send_email_and_get_id`. New helpers will be introduced to filter logs
according to the MID constructed in this function.

* new filters for searching logs with MID

* use new filters (and sending) functions

* add new helper for asserting non-existence of log message

* use new filters in tests

* Apply suggestions from code review

- `_mid` / `MID` => `_msgid` / `MSG_ID`
- Revised documentation / tooltip comments

* Apply suggestions from code review

* fix tests

* use more distinct names for MSG_ID headers

* update `_filter_service_log` to not use `-i -E`

Moreover, I added a function to print the whole mail log. Appropriate
comments were added to this function to indicate that one should only
use this function when necessary.

* adjust helpers to new helper filter

* follow-up of previous commit

* add CHANGELOG entry

* Apply suggestions from code review

* chore: Update OAuth2 to use new log helper

* Apply suggestions from code review

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

* added explicit `_regexp` filters for logs

* Apply suggestions from code review

---------

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

test/tests/parallel/set1/spam_virus/rspamd_full.bats

@@ -45,16 +45,17 @@ function setup_file() {
 
   # We will send 4 emails:
   #   1. The first one should pass just fine
-  _send_email_and_get_id MAIL_ID_PASS
+  _send_email_with_msgid 'rspamd-test-email-pass'
   #   2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection)
   _send_spam --expect-rejection
   #   3. The third one should be rejected due to a virus (ClamAV EICAR pattern)
   # shellcheck disable=SC2016
-  _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \
+  _send_email_with_msgid 'rspamd-test-email-virus' --expect-rejection \
     --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
   #   4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header)
   #      ref: https://rspamd.com/doc/gtube_patterns.html
-  _send_email_and_get_id MAIL_ID_HEADER --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
+  _send_email_with_msgid 'rspamd-test-email-header' \
+    --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
 
   _run_in_container cat /var/log/mail.log
   assert_success
@@ -92,7 +93,7 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test 'service log exist and contains proper content' {
-  _service_log_should_contain_string 'rspamd' 'rspamd .* is loading configuration'
+  _service_log_should_contain_string_regexp 'rspamd' 'rspamd .* is loading configuration'
   _service_log_should_contain_string 'rspamd' 'lua module clickhouse is disabled in the configuration'
   _service_log_should_contain_string 'rspamd' 'lua module elastic is disabled in the configuration'
   _service_log_should_contain_string 'rspamd' 'lua module neural is disabled in the configuration'
@@ -108,33 +109,40 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test 'normal mail passes fine' {
-  _service_log_should_contain_string 'rspamd' 'F \(no action\)'
+  _service_log_should_contain_string 'rspamd' 'F (no action)'
 
-  _print_mail_log_for_id "${MAIL_ID_PASS}"
+  _print_mail_log_for_msgid 'rspamd-test-email-pass'
   assert_output --partial "stored mail into mailbox 'INBOX'"
 
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
 
 @test 'detects and rejects spam' {
-  _service_log_should_contain_string 'rspamd' 'S \(reject\)'
+  _service_log_should_contain_string 'rspamd' 'S (reject)'
   _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"'
 
-  _print_mail_log_for_id "${MAIL_ID_SPAM}"
+  _print_mail_log_of_queue_id_from_msgid 'dms-test-email-spam'
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 Gtube pattern'
 
+  _print_mail_log_for_msgid 'dms-test-email-spam'
+  refute_output --partial "stored mail into mailbox 'INBOX'"
+  assert_failure
+
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
 
 @test 'detects and rejects virus' {
-  _service_log_should_contain_string 'rspamd' 'T \(reject\)'
+  _service_log_should_contain_string 'rspamd' 'T (reject)'
   _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"'
 
-  _print_mail_log_for_id "${MAIL_ID_VIRUS}"
+  _print_mail_log_of_queue_id_from_msgid 'rspamd-test-email-virus'
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"'
+
+  _print_mail_log_for_msgid 'dms-test-email-spam'
   refute_output --partial "stored mail into mailbox 'INBOX'"
+  assert_failure
 
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
@@ -217,10 +225,10 @@ function teardown_file() { _default_teardown ; }
   _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]'
   assert_success
 
-  _service_log_should_contain_string 'rspamd' 'S \(add header\)'
+  _service_log_should_contain_string 'rspamd' 'S (add header)'
   _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"'
 
-  _print_mail_log_for_id "${MAIL_ID_HEADER}"
+  _print_mail_log_for_msgid 'rspamd-test-email-header'
   assert_output --partial "fileinto action: stored mail into mailbox 'Junk'"
 
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
@@ -265,10 +273,10 @@ function teardown_file() { _default_teardown ; }
   _nc_wrapper 'nc/rspamd_imap_move_to_junk.txt' '0.0.0.0 143'
   sleep 1 # wait for the transaction to finish
 
-  _run_in_container cat /var/log/mail/mail.log
-  assert_success
-  assert_output --partial 'imapsieve: Matched static mailbox rule [1]'
-  refute_output --partial 'imapsieve: Matched static mailbox rule [2]'
+  _service_log_should_contain_string 'mail' 'imapsieve: Matched static mailbox rule [1]'
+  _service_log_should_not_contain_string 'mail' 'imapsieve: Matched static mailbox rule [2]'
+
+  _show_complete_mail_log
   for LINE in "${LEARN_SPAM_LINES[@]}"; do
     assert_output --partial "${LINE}"
   done
@@ -279,9 +287,9 @@ function teardown_file() { _default_teardown ; }
   _nc_wrapper 'nc/rspamd_imap_move_to_inbox.txt' '0.0.0.0 143'
   sleep 1 # wait for the transaction to finish
 
-  _run_in_container cat /var/log/mail/mail.log
-  assert_success
-  assert_output --partial 'imapsieve: Matched static mailbox rule [2]'
+  _service_log_should_contain_string 'mail' 'imapsieve: Matched static mailbox rule [2]'
+
+  _show_complete_mail_log
   for LINE in "${LEARN_HAM_LINES[@]}"; do
     assert_output --partial "${LINE}"
   done