From 443941e68748472604f9a6ee983e94732a460e0b Mon Sep 17 00:00:00 2001 From: realizelol Date: Mon, 24 Jun 2024 09:07:12 +0200 Subject: [PATCH] [Rspamd] Delete overriding obsolete rspamd plugin (#5900) * [Dockerfiles] rspamd: Delete COPY of metadata_exporter.lua plugin * [Dockerfiles] rspamd: Delete metadata_exporter.lua plugin file * Dockerfile: changed way of installing rspamd (granular version) --------- Co-authored-by: DerLinkman --- data/Dockerfiles/rspamd/Dockerfile | 15 +- data/Dockerfiles/rspamd/metadata_exporter.lua | 632 ------------------ 2 files changed, 9 insertions(+), 638 deletions(-) delete mode 100644 data/Dockerfiles/rspamd/metadata_exporter.lua diff --git a/data/Dockerfiles/rspamd/Dockerfile b/data/Dockerfiles/rspamd/Dockerfile index b664691b7..07f0bc36b 100644 --- a/data/Dockerfiles/rspamd/Dockerfile +++ b/data/Dockerfiles/rspamd/Dockerfile @@ -2,6 +2,7 @@ FROM debian:bullseye-slim LABEL maintainer "The Infrastructure Company GmbH " ARG DEBIAN_FRONTEND=noninteractive +ARG RSPAMD_VER=rspamd_3.7.5-2~8c86c1676 ARG CODENAME=bullseye ENV LC_ALL C @@ -12,11 +13,14 @@ RUN apt-get update && apt-get install -y \ apt-transport-https \ dnsutils \ netcat \ - && apt-key adv --fetch-keys https://rspamd.com/apt-stable/gpg.key \ - && echo "deb https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \ - && apt-get update \ - && apt-get --no-install-recommends -y install rspamd redis-tools procps nano \ - && rm -rf /var/lib/apt/lists/* \ + wget \ + redis-tools \ + procps \ + nano \ + && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \ + && wget -P /tmp https://rspamd.com/apt-stable/pool/main/r/rspamd/${RSPAMD_VER}~${CODENAME}_${arch}.deb\ + && apt install -y /tmp/${RSPAMD_VER}~${CODENAME}_${arch}.deb \ + && rm -rf /var/lib/apt/lists/* /tmp/*\ && apt-get autoremove --purge \ && apt-get clean \ && mkdir -p /run/rspamd \ @@ -25,7 +29,6 @@ RUN apt-get update && apt-get install -y \ && sed -i 's/#analysis_keyword_table > 0/analysis_cat_table.macro_exist == "M"/g' /usr/share/rspamd/lualib/lua_scanners/oletools.lua COPY settings.conf /etc/rspamd/settings.conf -COPY metadata_exporter.lua /usr/share/rspamd/plugins/metadata_exporter.lua COPY set_worker_password.sh /set_worker_password.sh COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/data/Dockerfiles/rspamd/metadata_exporter.lua b/data/Dockerfiles/rspamd/metadata_exporter.lua deleted file mode 100644 index 48a5ffce3..000000000 --- a/data/Dockerfiles/rspamd/metadata_exporter.lua +++ /dev/null @@ -1,632 +0,0 @@ ---[[ -Copyright (c) 2016, Andrew Lewis -Copyright (c) 2016, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -if confighelp then - return -end - --- A plugin that pushes metadata (or whole messages) to external services - -local redis_params -local lua_util = require "lua_util" -local rspamd_http = require "rspamd_http" -local rspamd_util = require "rspamd_util" -local rspamd_logger = require "rspamd_logger" -local ucl = require "ucl" -local E = {} -local N = 'metadata_exporter' - -local settings = { - pusher_enabled = {}, - pusher_format = {}, - pusher_select = {}, - mime_type = 'text/plain', - defer = false, - mail_from = '', - mail_to = 'postmaster@localhost', - helo = 'rspamd', - email_template = [[From: "Rspamd" <$mail_from> -To: $mail_to -Subject: Spam alert -Date: $date -MIME-Version: 1.0 -Message-ID: <$our_message_id> -Content-type: text/plain; charset=utf-8 -Content-Transfer-Encoding: 8bit - -Authenticated username: $user -IP: $ip -Queue ID: $qid -SMTP FROM: $from -SMTP RCPT: $rcpt -MIME From: $header_from -MIME To: $header_to -MIME Date: $header_date -Subject: $header_subject -Message-ID: $message_id -Action: $action -Score: $score -Symbols: $symbols]], -} - -local function get_general_metadata(task, flatten, no_content) - local r = {} - local ip = task:get_from_ip() - if ip and ip:is_valid() then - r.ip = tostring(ip) - else - r.ip = 'unknown' - end - r.user = task:get_user() or 'unknown' - r.qid = task:get_queue_id() or 'unknown' - r.subject = task:get_subject() or 'unknown' - r.action = task:get_metric_action('default') - - local s = task:get_metric_score('default')[1] - r.score = flatten and string.format('%.2f', s) or s - - local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings") - if fuzzy and #fuzzy > 0 then - local fz = {} - for _,h in ipairs(fuzzy) do - table.insert(fz, h) - end - if not flatten then - r.fuzzy = fz - else - r.fuzzy = table.concat(fz, ', ') - end - else - r.fuzzy = 'unknown' - end - - local rcpt = task:get_recipients('smtp') - if rcpt then - local l = {} - for _, a in ipairs(rcpt) do - table.insert(l, a['addr']) - end - if not flatten then - r.rcpt = l - else - r.rcpt = table.concat(l, ', ') - end - else - r.rcpt = 'unknown' - end - local from = task:get_from('smtp') - if ((from or E)[1] or E).addr then - r.from = from[1].addr - else - r.from = 'unknown' - end - local syminf = task:get_symbols_all() - if flatten then - local l = {} - for _, sym in ipairs(syminf) do - local txt - if sym.options then - local topt = table.concat(sym.options, ', ') - txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' .. ' [' .. topt .. ']' - else - txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' - end - table.insert(l, txt) - end - r.symbols = table.concat(l, '\n\t') - else - r.symbols = syminf - end - local function process_header(name) - local hdr = task:get_header_full(name) - if hdr then - local l = {} - for _, h in ipairs(hdr) do - table.insert(l, h.decoded) - end - if not flatten then - return l - else - return table.concat(l, '\n') - end - else - return 'unknown' - end - end - if not no_content then - r.header_from = process_header('from') - r.header_to = process_header('to') - r.header_subject = process_header('subject') - r.header_date = process_header('date') - r.message_id = task:get_message_id() - end - return r -end - -local formatters = { - default = function(task) - return task:get_content(), {} - end, - email_alert = function(task, rule, extra) - local meta = get_general_metadata(task, true) - local display_emails = {} - local mail_targets = {} - meta.mail_from = rule.mail_from or settings.mail_from - local mail_rcpt = rule.mail_to or settings.mail_to - if type(mail_rcpt) ~= 'table' then - table.insert(display_emails, string.format('<%s>', mail_rcpt)) - table.insert(mail_targets, mail_rcpt) - else - for _, e in ipairs(mail_rcpt) do - table.insert(display_emails, string.format('<%s>', e)) - table.insert(mail_targets, mail_rcpt) - end - end - if rule.email_alert_sender then - local x = task:get_from('smtp') - if x and string.len(x[1].addr) > 0 then - table.insert(mail_targets, x) - table.insert(display_emails, string.format('<%s>', x[1].addr)) - end - end - if rule.email_alert_user then - local x = task:get_user() - if x then - table.insert(mail_targets, x) - table.insert(display_emails, string.format('<%s>', x)) - end - end - if rule.email_alert_recipients then - local x = task:get_recipients('smtp') - if x then - for _, e in ipairs(x) do - if string.len(e.addr) > 0 then - table.insert(mail_targets, e.addr) - table.insert(display_emails, string.format('<%s>', e.addr)) - end - end - end - end - meta.mail_to = table.concat(display_emails, ', ') - meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd' - meta.date = rspamd_util.time_to_string(rspamd_util.get_time()) - return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets} - end, - json = function(task) - return ucl.to_format(get_general_metadata(task), 'json-compact') - end -} - -local function is_spam(action) - return (action == 'reject' or action == 'add header' or action == 'rewrite subject') -end - -local selectors = { - default = function(task) - return true - end, - is_spam = function(task) - local action = task:get_metric_action('default') - return is_spam(action) - end, - is_spam_authed = function(task) - if not task:get_user() then - return false - end - local action = task:get_metric_action('default') - return is_spam(action) - end, - is_reject = function(task) - local action = task:get_metric_action('default') - return (action == 'reject') - end, - is_reject_authed = function(task) - if not task:get_user() then - return false - end - local action = task:get_metric_action('default') - return (action == 'reject') - end, -} - -local function maybe_defer(task, rule) - if rule.defer then - rspamd_logger.warnx(task, 'deferring message') - task:set_pre_result('soft reject', 'deferred', N) - end -end - -local pushers = { - redis_pubsub = function(task, formatted, rule) - local _,ret,upstream - local function redis_pub_cb(err) - if err then - rspamd_logger.errx(task, 'got error %s when publishing on server %s', - err, upstream:get_addr()) - return maybe_defer(task, rule) - end - return true - end - ret,_,upstream = rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_pub_cb, --callback - 'PUBLISH', -- command - {rule.channel, formatted} -- arguments - ) - if not ret then - rspamd_logger.errx(task, 'error connecting to redis') - maybe_defer(task, rule) - end - end, - http = function(task, formatted, rule) - local function http_callback(err, code) - if err then - rspamd_logger.errx(task, 'got error %s in http callback', err) - return maybe_defer(task, rule) - end - if code ~= 200 then - rspamd_logger.errx(task, 'got unexpected http status: %s', code) - return maybe_defer(task, rule) - end - return true - end - local hdrs = {} - if rule.meta_headers then - local gm = get_general_metadata(task, false, true) - local pfx = rule.meta_header_prefix or 'X-Rspamd-' - for k, v in pairs(gm) do - if type(v) == 'table' then - hdrs[pfx .. k] = ucl.to_format(v, 'json-compact') - else - hdrs[pfx .. k] = v - end - end - end - rspamd_http.request({ - task=task, - url=rule.url, - body=formatted, - callback=http_callback, - mime_type=rule.mime_type or settings.mime_type, - headers=hdrs, - }) - end, - send_mail = function(task, formatted, rule, extra) - local lua_smtp = require "lua_smtp" - local function sendmail_cb(ret, err) - if not ret then - rspamd_logger.errx(task, 'SMTP export error: %s', err) - maybe_defer(task, rule) - end - end - - lua_smtp.sendmail({ - task = task, - host = rule.smtp, - port = rule.smtp_port or settings.smtp_port or 25, - from = rule.mail_from or settings.mail_from, - recipients = extra.mail_targets or rule.mail_to or settings.mail_to, - helo = rule.helo or settings.helo, - timeout = rule.timeout or settings.timeout, - }, formatted, sendmail_cb) - end, -} - -local opts = rspamd_config:get_all_opt(N) -if not opts then return end -local process_settings = { - select = function(val) - selectors.custom = assert(load(val))() - end, - format = function(val) - formatters.custom = assert(load(val))() - end, - push = function(val) - pushers.custom = assert(load(val))() - end, - custom_push = function(val) - if type(val) == 'table' then - for k, v in pairs(val) do - pushers[k] = assert(load(v))() - end - end - end, - custom_select = function(val) - if type(val) == 'table' then - for k, v in pairs(val) do - selectors[k] = assert(load(v))() - end - end - end, - custom_format = function(val) - if type(val) == 'table' then - for k, v in pairs(val) do - formatters[k] = assert(load(v))() - end - end - end, - pusher_enabled = function(val) - if type(val) == 'string' then - if pushers[val] then - settings.pusher_enabled[val] = true - else - rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val) - end - elseif type(val) == 'table' then - for _, v in ipairs(val) do - if pushers[v] then - settings.pusher_enabled[v] = true - else - rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val) - end - end - end - end, -} -for k, v in pairs(opts) do - local f = process_settings[k] - if f then - f(opts[k]) - else - settings[k] = v - end -end -if type(settings.rules) ~= 'table' then - -- Legacy config - settings.rules = {} - if not next(settings.pusher_enabled) then - if pushers.custom then - rspamd_logger.infox(rspamd_config, 'Custom pusher implicitly enabled') - settings.pusher_enabled.custom = true - else - -- Check legacy options - if settings.url then - rspamd_logger.warnx(rspamd_config, 'HTTP pusher implicitly enabled') - settings.pusher_enabled.http = true - end - if settings.channel then - rspamd_logger.warnx(rspamd_config, 'Redis Pubsub pusher implicitly enabled') - settings.pusher_enabled.redis_pubsub = true - end - if settings.smtp and settings.mail_to then - rspamd_logger.warnx(rspamd_config, 'SMTP pusher implicitly enabled') - settings.pusher_enabled.send_mail = true - end - end - end - if not next(settings.pusher_enabled) then - rspamd_logger.errx(rspamd_config, 'No push backend enabled') - return - end - if settings.formatter then - settings.format = formatters[settings.formatter] - if not settings.format then - rspamd_logger.errx(rspamd_config, 'No such formatter: %s', settings.formatter) - return - end - end - if settings.selector then - settings.select = selectors[settings.selector] - if not settings.select then - rspamd_logger.errx(rspamd_config, 'No such selector: %s', settings.selector) - return - end - end - for k in pairs(settings.pusher_enabled) do - local formatter = settings.pusher_format[k] - local selector = settings.pusher_select[k] - if not formatter then - settings.pusher_format[k] = settings.formatter or 'default' - rspamd_logger.infox(rspamd_config, 'Using default formatter for %s pusher', k) - else - if not formatters[formatter] then - rspamd_logger.errx(rspamd_config, 'No such formatter: %s - disabling %s', formatter, k) - settings.pusher_enabled.k = nil - end - end - if not selector then - settings.pusher_select[k] = settings.selector or 'default' - rspamd_logger.infox(rspamd_config, 'Using default selector for %s pusher', k) - else - if not selectors[selector] then - rspamd_logger.errx(rspamd_config, 'No such selector: %s - disabling %s', selector, k) - settings.pusher_enabled.k = nil - end - end - end - if settings.pusher_enabled.redis_pubsub then - redis_params = rspamd_parse_redis_server(N) - if not redis_params then - rspamd_logger.errx(rspamd_config, 'No redis servers are specified') - settings.pusher_enabled.redis_pubsub = nil - else - local r = {} - r.backend = 'redis_pubsub' - r.channel = settings.channel - r.defer = settings.defer - r.selector = settings.pusher_select.redis_pubsub - r.formatter = settings.pusher_format.redis_pubsub - settings.rules[r.backend:upper()] = r - end - end - if settings.pusher_enabled.http then - if not settings.url then - rspamd_logger.errx(rspamd_config, 'No URL is specified') - settings.pusher_enabled.http = nil - else - local r = {} - r.backend = 'http' - r.url = settings.url - r.mime_type = settings.mime_type - r.defer = settings.defer - r.selector = settings.pusher_select.http - r.formatter = settings.pusher_format.http - settings.rules[r.backend:upper()] = r - end - end - if settings.pusher_enabled.send_mail then - if not (settings.mail_to and settings.smtp) then - rspamd_logger.errx(rspamd_config, 'No mail_to and/or smtp setting is specified') - settings.pusher_enabled.send_mail = nil - else - local r = {} - r.backend = 'send_mail' - r.mail_to = settings.mail_to - r.mail_from = settings.mail_from - r.helo = settings.hello - r.smtp = settings.smtp - r.smtp_port = settings.smtp_port - r.email_template = settings.email_template - r.defer = settings.defer - r.selector = settings.pusher_select.send_mail - r.formatter = settings.pusher_format.send_mail - settings.rules[r.backend:upper()] = r - end - end - if not next(settings.pusher_enabled) then - rspamd_logger.errx(rspamd_config, 'No push backend enabled') - return - end -elseif not next(settings.rules) then - lua_util.debugm(N, rspamd_config, 'No rules enabled') - return -end -if not settings.rules or not next(settings.rules) then - rspamd_logger.errx(rspamd_config, 'No rules enabled') - return -end -local backend_required_elements = { - http = { - 'url', - }, - smtp = { - 'mail_to', - 'smtp', - }, - redis_pubsub = { - 'channel', - }, -} -local check_element = { - selector = function(k, v) - if not selectors[v] then - rspamd_logger.errx(rspamd_config, 'Rule %s has invalid selector %s', k, v) - return false - else - return true - end - end, - formatter = function(k, v) - if not formatters[v] then - rspamd_logger.errx(rspamd_config, 'Rule %s has invalid formatter %s', k, v) - return false - else - return true - end - end, -} -local backend_check = { - default = function(k, rule) - local reqset = backend_required_elements[rule.backend] - if reqset then - for _, e in ipairs(reqset) do - if not rule[e] then - rspamd_logger.errx(rspamd_config, 'Rule %s misses required setting %s', k, e) - settings.rules[k] = nil - end - end - end - for sett, v in pairs(rule) do - local f = check_element[sett] - if f then - if not f(sett, v) then - settings.rules[k] = nil - end - end - end - end, -} -backend_check.redis_pubsub = function(k, rule) - if not redis_params then - redis_params = rspamd_parse_redis_server(N) - end - if not redis_params then - rspamd_logger.errx(rspamd_config, 'No redis servers are specified') - settings.rules[k] = nil - else - backend_check.default(k, rule) - end -end -setmetatable(backend_check, { - __index = function() - return backend_check.default - end, -}) -for k, v in pairs(settings.rules) do - if type(v) == 'table' then - local backend = v.backend - if not backend then - rspamd_logger.errx(rspamd_config, 'Rule %s has no backend', k) - settings.rules[k] = nil - elseif not pushers[backend] then - rspamd_logger.errx(rspamd_config, 'Rule %s has invalid backend %s', k, backend) - settings.rules[k] = nil - else - local f = backend_check[backend] - f(k, v) - end - else - rspamd_logger.errx(rspamd_config, 'Rule %s has bad type: %s', k, type(v)) - settings.rules[k] = nil - end -end - -local function gen_exporter(rule) - return function (task) - if task:has_flag('skip') then return end - local selector = rule.selector or 'default' - local selected = selectors[selector](task) - if selected then - lua_util.debugm(N, task, 'Message selected for processing') - local formatter = rule.formatter or 'default' - local formatted, extra = formatters[formatter](task, rule) - if formatted then - pushers[rule.backend](task, formatted, rule, extra) - else - lua_util.debugm(N, task, 'Formatter [%s] returned non-truthy value [%s]', formatter, formatted) - end - else - lua_util.debugm(N, task, 'Selector [%s] returned non-truthy value [%s]', selector, selected) - end - end -end - -if not next(settings.rules) then - rspamd_logger.errx(rspamd_config, 'No rules enabled') - lua_util.disable_module(N, "config") -end -for k, r in pairs(settings.rules) do - rspamd_config:register_symbol({ - name = 'EXPORT_METADATA_' .. k, - type = 'idempotent', - callback = gen_exporter(r), - priority = 10, - flags = 'empty,explicit_disable,ignore_passthrough', - }) -end