diff options
Diffstat (limited to 'lualib/lua_scanners')
| -rw-r--r-- | lualib/lua_scanners/avast.lua | 304 | ||||
| -rw-r--r-- | lualib/lua_scanners/clamav.lua | 193 | ||||
| -rw-r--r-- | lualib/lua_scanners/cloudmark.lua | 372 | ||||
| -rw-r--r-- | lualib/lua_scanners/common.lua | 539 | ||||
| -rw-r--r-- | lualib/lua_scanners/dcc.lua | 313 | ||||
| -rw-r--r-- | lualib/lua_scanners/fprot.lua | 181 | ||||
| -rw-r--r-- | lualib/lua_scanners/icap.lua | 713 | ||||
| -rw-r--r-- | lualib/lua_scanners/init.lua | 75 | ||||
| -rw-r--r-- | lualib/lua_scanners/kaspersky_av.lua | 197 | ||||
| -rw-r--r-- | lualib/lua_scanners/kaspersky_se.lua | 287 | ||||
| -rw-r--r-- | lualib/lua_scanners/oletools.lua | 369 | ||||
| -rw-r--r-- | lualib/lua_scanners/p0f.lua | 227 | ||||
| -rw-r--r-- | lualib/lua_scanners/pyzor.lua | 206 | ||||
| -rw-r--r-- | lualib/lua_scanners/razor.lua | 181 | ||||
| -rw-r--r-- | lualib/lua_scanners/savapi.lua | 261 | ||||
| -rw-r--r-- | lualib/lua_scanners/sophos.lua | 192 | ||||
| -rw-r--r-- | lualib/lua_scanners/spamassassin.lua | 213 | ||||
| -rw-r--r-- | lualib/lua_scanners/vadesecure.lua | 351 | ||||
| -rw-r--r-- | lualib/lua_scanners/virustotal.lua | 214 |
19 files changed, 5388 insertions, 0 deletions
diff --git a/lualib/lua_scanners/avast.lua b/lualib/lua_scanners/avast.lua new file mode 100644 index 0000000..7e77897 --- /dev/null +++ b/lualib/lua_scanners/avast.lua @@ -0,0 +1,304 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module avast +-- This module contains avast av access functions +--]] + +local lua_util = require "lua_util" +local rspamd_util = require "rspamd_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_regexp = require "rspamd_regexp" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "avast" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function avast_config(opts) + local avast_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + timeout = 4.0, -- FIXME: this will break task_timeout! + log_clean = false, + detection_category = "virus", + retransmits = 1, + servers = nil, -- e.g. /var/run/avast/scan.sock + cache_expire = 3600, -- expire redis in one hour + message = default_message, + tmpdir = '/tmp', + } + + avast_conf = lua_util.override_defaults(avast_conf, opts) + + if not avast_conf.prefix then + avast_conf.prefix = 'rs_' .. avast_conf.name .. '_' + end + + if not avast_conf.log_prefix then + if avast_conf.name:lower() == avast_conf.type:lower() then + avast_conf.log_prefix = avast_conf.name + else + avast_conf.log_prefix = avast_conf.name .. ' (' .. avast_conf.type .. ')' + end + end + + if not avast_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers/unix socket defined') + + return nil + end + + avast_conf['upstreams'] = upstream_list.create(rspamd_config, + avast_conf['servers'], + 0) + + if avast_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', avast_conf.name) + return avast_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + avast_conf['servers']) + return nil +end + +local function avast_check(task, content, digest, rule, maybe_part) + local function avast_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local CRLF = '\r\n' + + -- Common tcp options + local tcp_opts = { + stop_pattern = CRLF, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout, + task = task + } + + -- Regexps to process reply from avast + local clean_re = rspamd_regexp.create_cached( + [=[(?!\\)\t\[\+\]]=] + ) + local virus_re = rspamd_regexp.create_cached( + [[(?!\\)\t\[L\]\d\.\d\t\d\s(.*)]] + ) + local error_re = rspamd_regexp.create_cached( + [[(?!\\)\t\[E\]\d+\.0\tError\s\d+\s(.*)]] + ) + + -- Used to make a dialog + local tcp_conn + + -- Save content in file as avast can work with files only + local fname = string.format('%s/%s.avtmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for avast scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure file cleanup on task processed + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + -- Dialog stages closures + local avast_helo_cb + local avast_scan_cb + local avast_scan_done_cb + + -- Utility closures + local function maybe_retransmit() + if retransmits > 0 then + retransmits = retransmits - 1 + else + rspamd_logger.errx(task, + '%s [%s]: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', + 0.0, 'fail', maybe_part) + + return + end + + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + tcp_opts.upstream = upstream + tcp_opts.callback = avast_helo_cb + + local is_succ, err = tcp.request(tcp_opts) + + if not is_succ then + rspamd_logger.infox(task, 'cannot create connection to avast server: %s (%s)', + addr:to_string(true), err) + else + lua_util.debugm(rule.log_prefix, task, 'established connection to %s; retransmits=%s', + addr:to_string(true), retransmits) + end + end + + local function no_connection_error(err) + if err then + if tcp_conn then + tcp_conn:close() + tcp_conn = nil + + rspamd_logger.infox(task, 'failed to request to avast (%s): %s', + addr:to_string(true), err) + maybe_retransmit() + end + + return false + end + + return true + end + + + -- Define callbacks + avast_helo_cb = function(merr, mdata, conn) + -- Called when we have established a connection but not read anything + tcp_conn = conn + + if no_connection_error(merr) then + -- Check mdata to ensure that it starts with 220 + if #mdata > 3 and tostring(mdata:span(1, 3)) == '220' then + tcp_conn:add_write(avast_scan_cb, string.format( + 'SCAN %s%s', fname, CRLF)) + else + rspamd_logger.errx(task, 'Unhandled response: %s', mdata) + end + end + end + + avast_scan_cb = function(merr) + -- Called when we have send request to avast and are waiting for reply + if no_connection_error(merr) then + tcp_conn:add_read(avast_scan_done_cb, CRLF) + end + end + + avast_scan_done_cb = function(merr, mdata) + if no_connection_error(merr) then + lua_util.debugm(rule.log_prefix, task, 'got reply from avast: %s', + mdata) + if #mdata > 4 then + local beg = tostring(mdata:span(1, 3)) + + if beg == '210' then + -- Ignore 210, fire another read + if tcp_conn then + tcp_conn:add_read(avast_scan_done_cb, CRLF) + end + elseif beg == '200' then + -- Final line + if tcp_conn then + tcp_conn:close() + tcp_conn = nil + end + else + -- Check line using regular expressions + local cached + local ret = clean_re:search(mdata, false, true) + + if ret then + cached = 'OK' + if rule.log_clean then + rspamd_logger.infox(task, + '%s [%s]: message or mime_part is clean', + rule.symbol, rule.type) + end + end + + if not cached then + ret = virus_re:search(mdata, false, true) + + if ret then + local vname = ret[1][2] + + if vname then + vname = vname:gsub('\\ ', ' '):gsub('\\\\', '\\') + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + cached = vname + end + end + end + + if not cached then + ret = error_re:search(mdata, false, true) + + if ret then + rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, ret[1][2]) + common.yield_result(task, rule, 'error:' .. ret[1][2], + 0.0, 'fail', maybe_part) + end + end + + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + else + -- Unexpected reply + rspamd_logger.errx(task, '%s: unexpected reply: %s', rule.log_prefix, mdata) + end + -- Read more + if tcp_conn then + tcp_conn:add_read(avast_scan_done_cb, CRLF) + end + end + end + end + end + + -- Send the real request + maybe_retransmit() + end + + if common.condition_check_and_continue(task, content, rule, digest, + avast_check_uncached, maybe_part) then + return + else + avast_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'Avast antivirus', + configure = avast_config, + check = avast_check, + name = N +} diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua new file mode 100644 index 0000000..fc99ab0 --- /dev/null +++ b/lualib/lua_scanners/clamav.lua @@ -0,0 +1,193 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module clamav +-- This module contains clamav access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_util = require "rspamd_util" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "clamav" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function clamav_config(opts) + local clamav_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + default_port = 3310, + log_clean = false, + timeout = 5.0, -- FIXME: this will break task_timeout! + detection_category = "virus", + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + } + + clamav_conf = lua_util.override_defaults(clamav_conf, opts) + + if not clamav_conf.prefix then + clamav_conf.prefix = 'rs_' .. clamav_conf.name .. '_' + end + + if not clamav_conf.log_prefix then + if clamav_conf.name:lower() == clamav_conf.type:lower() then + clamav_conf.log_prefix = clamav_conf.name + else + clamav_conf.log_prefix = clamav_conf.name .. ' (' .. clamav_conf.type .. ')' + end + end + + if not clamav_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + clamav_conf['upstreams'] = upstream_list.create(rspamd_config, + clamav_conf['servers'], + clamav_conf.default_port) + + if clamav_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', clamav_conf.name) + return clamav_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + clamav_conf['servers']) + return nil +end + +local function clamav_check(task, content, digest, rule, maybe_part) + local function clamav_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0", + #content) + local footer = rspamd_util.pack(">I4", 0) + + local function clamav_callback(err, data) + if err then + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = clamav_callback, + data = { header, content, footer }, + stop_pattern = '\0' + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix) + common.yield_result(task, rule, + 'failed to scan and retransmits exceed', 0.0, 'fail', + maybe_part) + end + + else + data = tostring(data) + local cached + lua_util.debugm(rule.name, task, '%s: got reply: %s', + rule.log_prefix, data) + if data == 'stream: OK' then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: message or mime_part is clean', + rule.log_prefix) + else + lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', rule.log_prefix) + end + else + local vname = string.match(data, 'stream: (.+) FOUND') + if string.find(vname, '^Heuristics%.Encrypted') then + rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) + common.yield_result(task, rule, 'File is encrypted: ' .. vname, + 0.0, 'encrypted', maybe_part) + cached = 'ENCRYPTED' + elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then + rspamd_logger.errx(task, '%s: ClamAV Found an OLE2 Office Macro', rule.log_prefix) + common.yield_result(task, rule, vname, 0.0, 'macro', maybe_part) + cached = 'MACRO' + elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then + rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix) + common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0, + 'fail', maybe_part) + elseif vname then + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + cached = vname + else + rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data) + common.yield_result(task, rule, 'unhandled response:' .. vname, 0.0, + 'fail', maybe_part) + end + end + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = clamav_callback, + upstream = upstream, + data = { header, content, footer }, + stop_pattern = '\0' + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, + clamav_check_uncached, maybe_part) then + return + else + clamav_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'clamav antivirus', + configure = clamav_config, + check = clamav_check, + name = N +} diff --git a/lualib/lua_scanners/cloudmark.lua b/lualib/lua_scanners/cloudmark.lua new file mode 100644 index 0000000..b07f238 --- /dev/null +++ b/lualib/lua_scanners/cloudmark.lua @@ -0,0 +1,372 @@ +--[[ +Copyright (c) 2021, Alexander Moisseev <moiseev@mezonplus.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module cloudmark +-- This module contains Cloudmark v2 interface +--]] + +local lua_util = require "lua_util" +local http = require "rspamd_http" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local ucl = require "ucl" +local rspamd_util = require "rspamd_util" +local common = require "lua_scanners/common" +local fun = require "fun" +local lua_mime = require "lua_mime" + +local N = 'cloudmark' +-- Boundary for multipart transfers, generated on module init +local static_boundary = rspamd_util.random_hex(32) + +local function cloudmark_url(rule, addr, maybe_url) + local url + local port = addr:get_port() + + maybe_url = maybe_url or rule.url + if port == 0 then + port = rule.default_port + end + if rule.use_https then + url = string.format('https://%s:%d%s', tostring(addr), + port, maybe_url) + else + url = string.format('http://%s:%d%s', tostring(addr), + port, maybe_url) + end + + return url +end + +-- Detect cloudmark max size +local function cloudmark_preload(rule, cfg, ev_base, _) + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local function max_message_size_cb(http_err, code, body, _) + if http_err then + rspamd_logger.errx(ev_base, 'HTTP error when getting max message size: %s', + http_err) + return + end + if code ~= 200 then + rspamd_logger.errx(ev_base, 'bad HTTP code when getting max message size: %s', code) + end + local parser = ucl.parser() + local ret, err = parser:parse_string(body) + if not ret then + rspamd_logger.errx(ev_base, 'could not parse response body [%s]: %s', body, err) + return + end + local obj = parser:get_object() + local ms = obj.maxMessageSize + if not ms then + rspamd_logger.errx(ev_base, 'missing maxMessageSize in the response body (JSON): %s', obj) + return + end + + rule.max_size = ms + lua_util.debugm(N, cfg, 'set maximum message size set to %s bytes', ms) + end + http.request({ + ev_base = ev_base, + config = cfg, + url = cloudmark_url(rule, addr, '/score/v2/max-message-size'), + callback = max_message_size_cb, + }) +end + +local function cloudmark_config(opts) + + local cloudmark_conf = { + name = N, + default_port = 2713, + url = '/score/v2/message', + use_https = false, + timeout = 5.0, + log_clean = false, + retransmits = 1, + score_threshold = 90, -- minimum score to considerate reply + message = '${SCANNER}: spam message found: "${VIRUS}"', + max_message = 0, + detection_category = "hash", + default_score = 1, + action = false, + log_spamcause = true, + symbol_fail = 'CLOUDMARK_FAIL', + symbol = 'CLOUDMARK_CHECK', + symbol_spam = 'CLOUDMARK_SPAM', + add_headers = false, -- allow addition of the headers from Cloudmark + } + + cloudmark_conf = lua_util.override_defaults(cloudmark_conf, opts) + + if not cloudmark_conf.prefix then + cloudmark_conf.prefix = 'rs_' .. cloudmark_conf.name .. '_' + end + + if not cloudmark_conf.log_prefix then + if cloudmark_conf.name:lower() == cloudmark_conf.type:lower() then + cloudmark_conf.log_prefix = cloudmark_conf.name + else + cloudmark_conf.log_prefix = cloudmark_conf.name .. ' (' .. cloudmark_conf.type .. ')' + end + end + + if not cloudmark_conf.servers and cloudmark_conf.socket then + cloudmark_conf.servers = cloudmark_conf.socket + end + + if not cloudmark_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + cloudmark_conf.upstreams = upstream_list.create(rspamd_config, + cloudmark_conf.servers, + cloudmark_conf.default_port) + + if cloudmark_conf.upstreams then + + cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } } + cloudmark_conf.preloads = { cloudmark_preload } + lua_util.add_debug_alias('external_services', cloudmark_conf.name) + return cloudmark_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + cloudmark_conf['servers']) + return nil +end + +-- Converts a key-value map to the table representing multipart body, with the following values: +-- `data`: data of the part +-- `filename`: optional filename +-- `content-type`: content type of the element (optional) +-- `content-transfer-encoding`: optional CTE header +local function table_to_multipart_body(tbl, boundary) + local seen_data = false + local out = {} + + for k, v in pairs(tbl) do + if v.data then + seen_data = true + table.insert(out, string.format('--%s\r\n', boundary)) + if v.filename then + table.insert(out, + string.format('Content-Disposition: form-data; name="%s"; filename="%s"\r\n', + k, v.filename)) + else + table.insert(out, + string.format('Content-Disposition: form-data; name="%s"\r\n', k)) + end + if v['content-type'] then + table.insert(out, + string.format('Content-Type: %s\r\n', v['content-type'])) + else + table.insert(out, 'Content-Type: text/plain\r\n') + end + if v['content-transfer-encoding'] then + table.insert(out, + string.format('Content-Transfer-Encoding: %s\r\n', + v['content-transfer-encoding'])) + else + table.insert(out, 'Content-Transfer-Encoding: binary\r\n') + end + table.insert(out, '\r\n') + table.insert(out, v.data) + table.insert(out, '\r\n') + end + end + + if seen_data then + table.insert(out, string.format('--%s--\r\n', boundary)) + end + + return out +end + +local function parse_cloudmark_reply(task, rule, body) + local parser = ucl.parser() + local ret, err = parser:parse_string(body) + if not ret then + rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body) + task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err) + return + end + local obj = parser:get_object() + lua_util.debugm(N, task, 'cloudmark reply is: %s', obj) + + if not obj.score then + rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body) + task:insert_result(rule.symbol_fail, 1.0, 'Parser error: no score') + return + end + + if obj.analysis then + -- Report analysis string + rspamd_logger.infox(task, 'cloudmark report string: %s', obj.analysis) + end + + local score = tonumber(obj.score) or 0 + if score >= rule.score_threshold then + task:insert_result(rule.symbol_spam, 1.0, tostring(score)) + end + + if rule.add_headers and type(obj.appendHeaders) == 'table' then + local headers_add = fun.tomap(fun.map(function(h) + return h.headerField, { + order = 1, value = h.body + } + end, obj.appendHeaders)) + lua_mime.modify_headers(task, { + add = headers_add + }) + end + +end + +local function cloudmark_check(task, content, digest, rule, maybe_part) + local function cloudmark_check_uncached() + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + local url = cloudmark_url(rule, addr) + local message_data = task:get_content() + if rule.max_message and rule.max_message > 0 and #message_data > rule.max_message then + task:insert_result(rule['symbol_fail'], 0.0, 'Message too large: ' .. #message_data) + return + end + local request = { + rfc822 = { + ['Content-Type'] = 'message/rfc822', + data = message_data, + } + } + + local helo = task:get_helo() + if helo then + request['heloDomain'] = { + data = helo, + } + end + local mail_from = task:get_from('smtp') or {} + if mail_from[1] and #mail_from[1].addr > 1 then + request['mailFrom'] = { + data = mail_from[1].addr + } + end + + local rcpt_to = task:get_recipients('smtp') + if rcpt_to then + request['rcptTo'] = { + data = table.concat(fun.totable(fun.map(function(r) + return r.addr + end, rcpt_to)), ',') + } + end + + local fip = task:get_from_ip() + if fip and fip:is_valid() then + request['connIp'] = tostring(fip) + end + + local hostname = task:get_hostname() + if hostname then + request['fromHost'] = hostname + end + + local request_data = { + task = task, + url = url, + body = table_to_multipart_body(request, static_boundary), + headers = { + ['Content-Type'] = string.format('multipart/form-data; boundary="%s"', static_boundary) + }, + timeout = rule.timeout, + } + + local function cloudmark_callback(http_err, code, body, headers) + + local function cloudmark_requery() + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, + '%s: request Error: %s - retries left: %s', + rule.log_prefix, http_err, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + url = cloudmark_url(rule, addr) + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + request_data.url = url + + http.request(request_data) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. + 'retransmits exceed') + upstream:fail() + end + end + + if http_err then + cloudmark_requery() + else + -- Parse the response + if upstream then + upstream:ok() + end + if code ~= 200 then + rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) + return + end + parse_cloudmark_reply(task, rule, body) + end + end + + request_data.callback = cloudmark_callback + http.request(request_data) + end + + if common.condition_check_and_continue(task, content, rule, digest, + cloudmark_check_uncached, maybe_part) then + return + else + cloudmark_check_uncached() + end +end + +return { + type = { 'cloudmark', 'scanner' }, + description = 'Cloudmark cartridge interface', + configure = cloudmark_config, + check = cloudmark_check, + name = N, +} diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua new file mode 100644 index 0000000..11f5e1f --- /dev/null +++ b/lualib/lua_scanners/common.lua @@ -0,0 +1,539 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_scanners_common +-- This module contains common external scanners functions +--]] + +local rspamd_logger = require "rspamd_logger" +local rspamd_regexp = require "rspamd_regexp" +local lua_util = require "lua_util" +local lua_redis = require "lua_redis" +local lua_magic_types = require "lua_magic/types" +local fun = require "fun" + +local exports = {} + +local function log_clean(task, rule, msg) + + msg = msg or 'message or mime_part is clean' + + if rule.log_clean then + rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg) + else + lua_util.debugm(rule.name, task, '%s: %s', rule.log_prefix, msg) + end + +end + +local function match_patterns(default_sym, found, patterns, dyn_weight) + if type(patterns) ~= 'table' then + return default_sym, dyn_weight + end + if not patterns[1] then + for sym, pat in pairs(patterns) do + if pat:match(found) then + return sym, '1' + end + end + return default_sym, dyn_weight + else + for _, p in ipairs(patterns) do + for sym, pat in pairs(p) do + if pat:match(found) then + return sym, '1' + end + end + end + return default_sym, dyn_weight + end +end + +local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) + local all_whitelisted = true + local patterns + local symbol + local threat_table + local threat_info + local flags + + if type(vname) == 'string' then + threat_table = { vname } + elseif type(vname) == 'table' then + threat_table = vname + end + + + -- This should be more generic + if not is_fail then + patterns = rule.patterns + symbol = rule.symbol + threat_info = rule.detection_category .. 'found' + if not dyn_weight then + dyn_weight = 1.0 + end + elseif is_fail == 'fail' then + patterns = rule.patterns_fail + symbol = rule.symbol_fail + threat_info = "FAILED with error" + dyn_weight = 0.0 + elseif is_fail == 'encrypted' then + patterns = rule.patterns + symbol = rule.symbol_encrypted + threat_info = "Scan has returned that input was encrypted" + dyn_weight = 1.0 + elseif is_fail == 'macro' then + patterns = rule.patterns + symbol = rule.symbol_macro + threat_info = "Scan has returned that input contains macros" + dyn_weight = 1.0 + end + + for _, tm in ipairs(threat_table) do + local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight) + if rule.whitelist and rule.whitelist:get_key(tm) then + rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, tm) + else + all_whitelisted = false + rspamd_logger.infox(task, '%s: result - %s: "%s - score: %s"', + rule.log_prefix, threat_info, tm, symscore) + + if maybe_part and rule.show_attachments and maybe_part:get_filename() then + local fname = maybe_part:get_filename() + task:insert_result(symname, symscore, string.format("%s|%s", + tm, fname)) + else + task:insert_result(symname, symscore, tm) + end + + end + end + + if rule.action and is_fail ~= 'fail' and not all_whitelisted then + threat_table = table.concat(threat_table, '; ') + if rule.action ~= 'reject' then + flags = 'least' + end + task:set_pre_result(rule.action, + lua_util.template(rule.message or 'Rejected', { + SCANNER = rule.name, + VIRUS = threat_table, + }), rule.name, nil, nil, flags) + end +end + +local function message_not_too_large(task, content, rule) + local max_size = tonumber(rule.max_size) + if not max_size then + return true + end + if #content > max_size then + rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)", + rule.log_prefix, #content, max_size) + return false + end + return true +end + +local function message_not_too_small(task, content, rule) + local min_size = tonumber(rule.min_size) + if not min_size then + return true + end + if #content < min_size then + rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)", + rule.log_prefix, #content, min_size) + return false + end + return true +end + +local function message_min_words(task, rule) + if rule.text_part_min_words and tonumber(rule.text_part_min_words) > 0 then + local text_part_above_limit = false + local text_parts = task:get_text_parts() + + local filter_func = function(p) + return p:get_words_count() >= tonumber(rule.text_part_min_words) + end + + fun.each(function(p) + text_part_above_limit = true + end, fun.filter(filter_func, text_parts)) + + if not text_part_above_limit then + rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s', + rule.log_prefix, rule.text_part_min_words) + end + + return text_part_above_limit + else + return true + end +end + +local function dynamic_scan(task, rule) + if rule.dynamic_scan then + if rule.action ~= 'reject' then + local metric_result = task:get_metric_score() + local metric_action = task:get_metric_action() + local has_pre_result = task:has_pre_result() + -- ToDo: needed? + -- Sometimes leads to FPs + --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then + -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject") + -- return false + --elseif metric_result[1] > metric_result[2]*2 then + if metric_result[1] > metric_result[2] * 2 then + rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1]) + return false + elseif has_pre_result and metric_action == 'reject' then + rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set') + return false + else + return true, 'undecided' + end + else + return true, 'dynamic_scan is not possible with config `action=reject;`' + end + else + return true + end +end + +local function need_check(task, content, rule, digest, fn, maybe_part) + + local uncached = true + local key = digest + + local function redis_av_cb(err, data) + if data and type(data) == 'string' then + -- Cached + data = lua_util.str_split(data, '\t') + local threat_string = lua_util.str_split(data[1], '\v') + local score = data[2] or rule.default_score + + if threat_string[1] ~= 'OK' then + if threat_string[1] == 'MACRO' then + yield_result(task, rule, 'File contains macros', + 0.0, 'macro', maybe_part) + elseif threat_string[1] == 'ENCRYPTED' then + yield_result(task, rule, 'File is encrypted', + 0.0, 'encrypted', maybe_part) + else + lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s', + rule.log_prefix, key, threat_string[1], score) + yield_result(task, rule, threat_string, score, false, maybe_part) + end + + else + lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s', + rule.log_prefix, key, threat_string[1]) + end + uncached = false + else + if err then + rspamd_logger.errx(task, 'got error checking cache: %s', err) + end + end + + local f_message_not_too_large = message_not_too_large(task, content, rule) + local f_message_not_too_small = message_not_too_small(task, content, rule) + local f_message_min_words = message_min_words(task, rule) + local f_dynamic_scan = dynamic_scan(task, rule) + + if uncached and + f_message_not_too_large and + f_message_not_too_small and + f_message_min_words and + f_dynamic_scan then + + fn() + + end + + end + + if rule.redis_params and not rule.no_cache then + + key = rule.prefix .. key + + if lua_redis.redis_make_request(task, + rule.redis_params, -- connect params + key, -- hash key + false, -- is write + redis_av_cb, --callback + 'GET', -- command + { key } -- arguments) + ) then + return true + end + end + + return false + +end + +local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) + local key = digest + if not dyn_weight then + dyn_weight = 1.0 + end + + local function redis_set_cb(err) + -- Do nothing + if err then + rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s', + rule.detection_category, to_save, key, err) + else + lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s', + rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) + end + end + + if type(to_save) == 'table' then + to_save = table.concat(to_save, '\v') + end + + local value_tbl = { to_save, dyn_weight } + if maybe_part and rule.show_attachments and maybe_part:get_filename() then + local fname = maybe_part:get_filename() + table.insert(value_tbl, fname) + end + local value = table.concat(value_tbl, '\t') + + if rule.redis_params and rule.prefix then + key = rule.prefix .. key + + lua_redis.redis_make_request(task, + rule.redis_params, -- connect params + key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { key, rule.cache_expire or 0, value } + ) + end + + return false +end + +local function create_regex_table(patterns) + local regex_table = {} + if patterns[1] then + for i, p in ipairs(patterns) do + if type(p) == 'table' then + local new_set = {} + for k, v in pairs(p) do + new_set[k] = rspamd_regexp.create_cached(v) + end + regex_table[i] = new_set + else + regex_table[i] = {} + end + end + else + for k, v in pairs(patterns) do + regex_table[k] = rspamd_regexp.create_cached(v) + end + end + return regex_table +end + +local function match_filter(task, rule, found, patterns, pat_type) + if type(patterns) ~= 'table' or not found then + return false + end + if not patterns[1] then + for _, pat in pairs(patterns) do + if pat_type == 'ext' and tostring(pat) == tostring(found) then + return true + elseif pat_type == 'regex' and pat:match(found) then + return true + end + end + return false + else + for _, p in ipairs(patterns) do + for _, pat in ipairs(p) do + if pat_type == 'ext' and tostring(pat) == tostring(found) then + return true + elseif pat_type == 'regex' and pat:match(found) then + return true + end + end + end + return false + end +end + +-- borrowed from mime_types.lua +-- ext is the last extension, LOWERCASED +-- ext2 is the one before last extension LOWERCASED +local function gen_extension(fname) + local filename_parts = lua_util.str_split(fname, '.') + + local ext = {} + for n = 1, 2 do + ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil + end + return ext[1], ext[2], filename_parts +end + +local function check_parts_match(task, rule) + + local filter_func = function(p) + local mtype, msubtype = p:get_type() + local detected_ext = p:get_detected_ext() + local fname = p:get_filename() + local ext, ext2 + + if rule.scan_all_mime_parts == false then + -- check file extension and filename regex matching + --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname) + if fname ~= nil then + ext, ext2 = gen_extension(fname) + --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname) + if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') + or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then + lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2) + return true + elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then + lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix) + return true + end + end + -- check content type string regex matching + if mtype ~= nil and msubtype ~= nil then + local ct = string.format('%s/%s', mtype, msubtype):lower() + if match_filter(task, rule, ct, rule.mime_parts_filter_regex, 'regex') then + lua_util.debugm(rule.name, task, '%s: regex content-type: %s', rule.log_prefix, ct) + return true + end + end + -- check detected content type (libmagic) regex matching + if detected_ext then + local magic = lua_magic_types[detected_ext] or {} + if match_filter(task, rule, detected_ext, rule.mime_parts_filter_ext, 'ext') then + lua_util.debugm(rule.name, task, '%s: detected extension matched: |%s|', rule.log_prefix, detected_ext) + return true + elseif magic.ct and match_filter(task, rule, magic.ct, rule.mime_parts_filter_regex, 'regex') then + lua_util.debugm(rule.name, task, '%s: regex detected libmagic content-type: %s', + rule.log_prefix, magic.ct) + return true + end + end + -- check filenames in archives + if p:is_archive() then + local arch = p:get_archive() + local filelist = arch:get_files_full(1000) + for _, f in ipairs(filelist) do + ext, ext2 = gen_extension(f.name) + if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') + or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then + lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2) + --lua_util.debugm(rule.name, task, '%s: extension matched in archive: %s', rule.log_prefix, ext) + return true + elseif match_filter(task, rule, f.name, rule.mime_parts_filter_regex, 'regex') then + lua_util.debugm(rule.name, task, '%s: filename regex matched in archive', rule.log_prefix) + return true + end + end + end + end + + -- check text_part has more words than text_part_min_words_check + if rule.scan_text_mime and rule.text_part_min_words and p:is_text() and + p:get_words_count() >= tonumber(rule.text_part_min_words) then + return true + end + + if rule.scan_image_mime and p:is_image() then + return true + end + + if rule.scan_all_mime_parts ~= false then + local is_part_checkable = (p:is_attachment() and (not p:is_image() or rule.scan_image_mime)) + if detected_ext then + -- We know what to scan! + local magic = lua_magic_types[detected_ext] or {} + + if magic.av_check ~= false or is_part_checkable then + return true + end + elseif is_part_checkable then + -- Just rely on attachment property + return true + end + end + + return false + end + + return fun.filter(filter_func, task:get_parts()) +end + +local function check_metric_results(task, rule) + + if rule.action ~= 'reject' then + local metric_result = task:get_metric_score() + local metric_action = task:get_metric_action() + local has_pre_result = task:has_pre_result() + + if rule.symbol_type == 'postfilter' and metric_action == 'reject' then + return true, 'result is already reject' + elseif metric_result[1] > metric_result[2] * 2 then + return true, 'score > 2 * reject_level: ' .. metric_result[1] + elseif has_pre_result and metric_action == 'reject' then + return true, 'pre_result reject is set' + else + return false, 'undecided' + end + else + return false, 'dynamic_scan is not possible with config `action=reject;`' + end +end + +exports.log_clean = log_clean +exports.yield_result = yield_result +exports.match_patterns = match_patterns +exports.condition_check_and_continue = need_check +exports.save_cache = save_cache +exports.create_regex_table = create_regex_table +exports.check_parts_match = check_parts_match +exports.check_metric_results = check_metric_results + +setmetatable(exports, { + __call = function(t, override) + for k, v in pairs(t) do + if _G[k] ~= nil then + local msg = 'function ' .. k .. ' already exists in global scope.' + if override then + _G[k] = v + print('WARNING: ' .. msg .. ' Overwritten.') + else + print('NOTICE: ' .. msg .. ' Skipped.') + end + else + _G[k] = v + end + end + end, +}) + +return exports diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua new file mode 100644 index 0000000..8d5e9e1 --- /dev/null +++ b/lualib/lua_scanners/dcc.lua @@ -0,0 +1,313 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module dcc +-- This module contains dcc access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" +local fun = require "fun" + +local N = 'dcc' + +local function dcc_config(opts) + + local dcc_conf = { + name = N, + default_port = 10045, + timeout = 5.0, + log_clean = false, + retransmits = 2, + cache_expire = 7200, -- expire redis in 2h + message = '${SCANNER}: bulk message found: "${VIRUS}"', + detection_category = "hash", + default_score = 1, + action = false, + client = '0.0.0.0', + symbol_fail = 'DCC_FAIL', + symbol = 'DCC_REJECT', + symbol_bulk = 'DCC_BULK', + body_max = 999999, + fuz1_max = 999999, + fuz2_max = 999999, + } + + dcc_conf = lua_util.override_defaults(dcc_conf, opts) + + if not dcc_conf.prefix then + dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_' + end + + if not dcc_conf.log_prefix then + dcc_conf.log_prefix = dcc_conf.name + end + + if not dcc_conf.servers and dcc_conf.socket then + dcc_conf.servers = dcc_conf.socket + end + + if not dcc_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + dcc_conf.upstreams = upstream_list.create(rspamd_config, + dcc_conf.servers, + dcc_conf.default_port) + + if dcc_conf.upstreams then + lua_util.add_debug_alias('external_services', dcc_conf.name) + return dcc_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + dcc_conf['servers']) + return nil +end + +local function dcc_check(task, content, digest, rule) + local function dcc_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local client = rule.client + + local client_ip = task:get_from_ip() + if client_ip and client_ip:is_valid() then + client = client_ip:to_string() + end + local client_host = task:get_hostname() + if client_host then + client = client .. "\r" .. client_host + end + + -- HELO + local helo = task:get_helo() or '' + + -- Envelope From + local ef = task:get_from() + local envfrom = 'test@example.com' + if ef and ef[1] then + envfrom = ef[1]['addr'] + end + + -- Envelope To + local envrcpt = 'test@example.com' + local rcpts = task:get_recipients(); + if rcpts then + local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt) + return rcpt['addr'] + end, + rcpts)), '\n') + if dcc_recipients then + envrcpt = dcc_recipients + end + end + + -- Build the DCC query + -- https://www.dcc-servers.net/dcc/dcc-tree/dccifd.html#Protocol + local request_data = { + "header\n", + client .. "\n", + helo .. "\n", + envfrom .. "\n", + envrcpt .. "\n", + "\n", + content + } + + local function dcc_callback(err, data, conn) + + local function dcc_requery() + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule.timeout or 2.0, + upstream = upstream, + shutdown = true, + data = request_data, + callback = dcc_callback, + body_max = 999999, + fuz1_max = 999999, + fuz2_max = 999999, + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') + end + end + + if err then + + dcc_requery() + + else + -- Parse the response + local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$") + lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"', + result, disposition, header) + + if header then + -- Unfold header + header = header:gsub('\r?\n%s*', ' ') + local _, _, info = header:find("; (.-)$") + if (result == 'R') then + -- Reject + common.yield_result(task, rule, info, rule.default_score) + common.save_cache(task, digest, rule, info, rule.default_score) + elseif (result == 'T') then + -- Temporary failure + rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result) + dcc_requery() + elseif result == 'A' then + + local opts = {} + local score = 0.0 + info = info:lower() + local rep = info:match('rep=([^=%s]+)') + + -- Adjust reputation if available + if rep then + rep = tonumber(rep) + end + if not rep then + rep = 1.0 + end + + local function check_threshold(what, num, lim) + local rnum + if num == 'many' then + rnum = lim + else + rnum = tonumber(num) + end + + if rnum and rnum >= lim then + opts[#opts + 1] = string.format('%s=%s', what, num) + score = score + rep / 3.0 + end + end + + info = info:lower() + local body = info:match('body=([^=%s]+)') + + if body then + check_threshold('body', body, rule.body_max) + end + + local fuz1 = info:match('fuz1=([^=%s]+)') + + if fuz1 then + check_threshold('fuz1', fuz1, rule.fuz1_max) + end + + local fuz2 = info:match('fuz2=([^=%s]+)') + + if fuz2 then + check_threshold('fuz2', fuz2, rule.fuz2_max) + end + + if #opts > 0 and score > 0 then + task:insert_result(rule.symbol_bulk, + score, + opts) + common.save_cache(task, digest, rule, opts, score) + else + common.save_cache(task, digest, rule, 'OK') + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result A - info: %s', + rule.log_prefix, info) + else + lua_util.debugm(rule.name, task, '%s: returned result A - info: %s', + rule.log_prefix, info) + end + end + elseif result == 'G' then + -- do nothing + common.save_cache(task, digest, rule, 'OK') + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info) + else + lua_util.debugm(rule.name, task, '%s: returned result G - info: %s', rule.log_prefix, info) + end + elseif result == 'S' then + -- do nothing + common.save_cache(task, digest, rule, 'OK') + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info) + else + lua_util.debugm(rule.name, task, '%s: returned result S - info: %s', rule.log_prefix, info) + end + else + -- Unknown result + rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result); + common.yield_result(task, rule, 'error: ' .. result, 0.0, 'fail') + end + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule.timeout or 2.0, + shutdown = true, + upstream = upstream, + data = request_data, + callback = dcc_callback, + body_max = 999999, + fuz1_max = 999999, + fuz2_max = 999999, + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, dcc_check_uncached) then + return + else + dcc_check_uncached() + end + +end + +return { + type = { 'dcc', 'bulk', 'hash', 'scanner' }, + description = 'dcc bulk scanner', + configure = dcc_config, + check = dcc_check, + name = N +} diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua new file mode 100644 index 0000000..5a469c3 --- /dev/null +++ b/lualib/lua_scanners/fprot.lua @@ -0,0 +1,181 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module fprot +-- This module contains fprot access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "fprot" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function fprot_config(opts) + local fprot_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + default_port = 10200, + timeout = 5.0, -- FIXME: this will break task_timeout! + log_clean = false, + detection_category = "virus", + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + } + + fprot_conf = lua_util.override_defaults(fprot_conf, opts) + + if not fprot_conf.prefix then + fprot_conf.prefix = 'rs_' .. fprot_conf.name .. '_' + end + + if not fprot_conf.log_prefix then + if fprot_conf.name:lower() == fprot_conf.type:lower() then + fprot_conf.log_prefix = fprot_conf.name + else + fprot_conf.log_prefix = fprot_conf.name .. ' (' .. fprot_conf.type .. ')' + end + end + + if not fprot_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + fprot_conf['upstreams'] = upstream_list.create(rspamd_config, + fprot_conf['servers'], + fprot_conf.default_port) + + if fprot_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', fprot_conf.name) + return fprot_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + fprot_conf['servers']) + return nil +end + +local function fprot_check(task, content, digest, rule, maybe_part) + local function fprot_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local scan_id = task:get_queue_id() + if not scan_id then + scan_id = task:get_uid() + end + local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id, + #content) + local footer = '\n' + + local function fprot_callback(err, data) + if err then + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = fprot_callback, + data = { header, content, footer }, + stop_pattern = '\n' + }) + else + rspamd_logger.errx(task, + '%s [%s]: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', + 0.0, 'fail', maybe_part) + end + else + upstream:ok() + data = tostring(data) + local cached + local clean = string.match(data, '^0 <clean>') + if clean then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, + '%s [%s]: message or mime_part is clean', + rule['symbol'], rule['type']) + end + else + -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occurred + -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail + local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') + if not vname then + rspamd_logger.errx(task, 'Unhandled response: %s', data) + else + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + cached = vname + end + end + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = fprot_callback, + data = { header, content, footer }, + stop_pattern = '\n' + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, + fprot_check_uncached, maybe_part) then + return + else + fprot_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'fprot antivirus', + configure = fprot_config, + check = fprot_check, + name = N +} diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua new file mode 100644 index 0000000..682562d --- /dev/null +++ b/lualib/lua_scanners/icap.lua @@ -0,0 +1,713 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[ +@module icap +This module contains icap access functions. +Currently tested with + - C-ICAP Squidclamav / echo + - Checkpoint Sandblast + - F-Secure Internet Gatekeeper + - Kaspersky Web Traffic Security + - Kaspersky Scan Engine 2.0 + - McAfee Web Gateway 9/10/11 + - Sophos Savdi + - Symantec (Rspamd <3.2, >=3.2 untested) + - Trend Micro IWSVA 6.0 + - Trend Micro Web Gateway + +@TODO + - Preview / Continue + - Reqmod URL's + - Content-Type / Filename +]] -- + +--[[ +Configuration Notes: + +C-ICAP Squidclamav + scheme = "squidclamav"; + +Checkpoint Sandblast example: + scheme = "sandblast"; + +ESET Gateway Security / Antivirus for Linux example: + scheme = "scan"; + +F-Secure Internet Gatekeeper example: + scheme = "respmod"; + x_client_header = true; + x_rcpt_header = true; + x_from_header = true; + +Kaspersky Web Traffic Security example: + scheme = "av/respmod"; + x_client_header = true; + +Kaspersky Web Traffic Security (as configured in kavicapd.xml): + scheme = "resp"; + x_client_header = true; + +McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules) + scheme = "respmod"; + x_client_header = true; + +Sophos SAVDI example: + # scheme as configured in savdi.conf (name option in service section) + scheme = "respmod"; + +Symantec example: + scheme = "avscan"; + +Trend Micro IWSVA example (X-Virus-ID/X-Infection-Found headers must be activated): + scheme = "avscan"; + x_client_header = true; + +Trend Micro Web Gateway example (X-Virus-ID/X-Infection-Found headers must be activated): + scheme = "interscan"; + x_client_header = true; +]] -- + + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" +local rspamd_util = require "rspamd_util" +local rspamd_version = rspamd_version + +local N = 'icap' + +local function icap_config(opts) + + local icap_conf = { + name = N, + scan_mime_parts = true, + scan_all_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + scheme = "scan", + default_port = 1344, + ssl = false, + no_ssl_verify = false, + timeout = 10.0, + log_clean = false, + retransmits = 2, + cache_expire = 7200, -- expire redis in one hour + message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"', + detection_category = "virus", + default_score = 1, + action = false, + dynamic_scan = false, + user_agent = "Rspamd", + x_client_header = false, + x_rcpt_header = false, + x_from_header = false, + req_headers_enabled = true, + req_fake_url = "http://127.0.0.1/mail", + http_headers_enabled = true, + use_http_result_header = true, + use_http_3xx_as_threat = false, + use_specific_content_type = false, -- Use content type from a part where possible + } + + icap_conf = lua_util.override_defaults(icap_conf, opts) + + if not icap_conf.prefix then + icap_conf.prefix = 'rs_' .. icap_conf.name .. '_' + end + + if not icap_conf.log_prefix then + icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')' + end + + if not icap_conf.log_prefix then + if icap_conf.name:lower() == icap_conf.type:lower() then + icap_conf.log_prefix = icap_conf.name + else + icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')' + end + end + + if not icap_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + icap_conf.upstreams = upstream_list.create(rspamd_config, + icap_conf.servers, + icap_conf.default_port) + + if icap_conf.upstreams then + lua_util.add_debug_alias('external_services', icap_conf.name) + return icap_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + icap_conf.servers) + return nil +end + +local function icap_check(task, content, digest, rule, maybe_part) + local function icap_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local http_headers = {} + local req_headers = {} + local tcp_options = {} + local threat_table = {} + + -- Build extended User Agent + if rule.user_agent == "extended" then + rule.user_agent = string.format("Rspamd/%s-%s (%s/%s)", + rspamd_version('main'), + rspamd_version('id'), + rspamd_util.get_hostname(), + string.sub(task:get_uid(), 1, 6)) + end + + -- Build the icap queries + local options_request = { + string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), + string.format("User-Agent: %s\r\n", rule.user_agent), + "Connection: keep-alive\r\n", + "Encapsulated: null-body=0\r\n\r\n", + } + if rule.user_agent == "none" then + table.remove(options_request, 3) + end + + local respond_headers = { + -- Add main RESPMOD header before any other + string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), + } + + local size = tonumber(#content) + local chunked_size = string.format("%x", size) + + local function icap_callback(err, conn) + + local function icap_requery(err_m, info) + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, + '%s: %s Request Error: %s - retries left: %s', + rule.log_prefix, info, err_m, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + + tcp_options.host = addr:to_string() + tcp_options.port = addr:get_port() + tcp_options.callback = icap_callback + tcp_options.data = options_request + tcp_options.upstream = upstream + + tcp.request(tcp_options) + + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - error: %s', rule.log_prefix, err_m or '') + common.yield_result(task, rule, string.format('failed - error: %s', err_m), + 0.0, 'fail', maybe_part) + end + end + + local function get_req_headers() + + local in_client_ip = task:get_from_ip() + local req_hlen = 2 + if maybe_part then + table.insert(req_headers, + string.format('GET http://%s/%s HTTP/1.0\r\n', in_client_ip, maybe_part:get_filename())) + if rule.use_specific_content_type then + table.insert(http_headers, string.format('Content-Type: %s/%s\r\n', maybe_part:get_detected_type())) + --else + -- To test: what content type is better for icap servers? + --table.insert(http_headers, 'Content-Type: text/plain\r\n') + end + else + table.insert(req_headers, string.format('GET %s HTTP/1.0\r\n', rule.req_fake_url)) + table.insert(http_headers, string.format('Content-Type: application/octet-stream\r\n')) + end + table.insert(req_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time()))) + if rule.user_agent ~= "none" then + table.insert(req_headers, string.format("User-Agent: %s\r\n", rule.user_agent)) + end + + for _, h in ipairs(req_headers) do + req_hlen = req_hlen + tonumber(#h) + end + + return req_hlen, req_headers + + end + + local function get_http_headers() + local http_hlen = 2 + table.insert(http_headers, 'HTTP/1.0 200 OK\r\n') + table.insert(http_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time()))) + table.insert(http_headers, string.format('Server: %s\r\n', 'Apache/2.4')) + if rule.user_agent ~= "none" then + table.insert(http_headers, string.format("User-Agent: %s\r\n", rule.user_agent)) + end + --table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html')) + table.insert(http_headers, string.format('Content-Length: %s\r\n', size)) + + for _, h in ipairs(http_headers) do + http_hlen = http_hlen + tonumber(#h) + end + + return http_hlen, http_headers + + end + + local function get_respond_query() + local req_hlen = 0 + local resp_req_headers + local http_hlen = 0 + local resp_http_headers + + -- Append all extra headers + if rule.user_agent ~= "none" then + table.insert(respond_headers, + string.format("User-Agent: %s\r\n", rule.user_agent)) + end + + if rule.req_headers_enabled then + req_hlen, resp_req_headers = get_req_headers() + end + if rule.http_headers_enabled then + http_hlen, resp_http_headers = get_http_headers() + end + + if rule.req_headers_enabled and rule.http_headers_enabled then + local res_body_hlen = req_hlen + http_hlen + table.insert(respond_headers, + string.format('Encapsulated: req-hdr=0, res-hdr=%s, res-body=%s\r\n', + req_hlen, res_body_hlen)) + elseif rule.http_headers_enabled then + table.insert(respond_headers, + string.format('Encapsulated: res-hdr=0, res-body=%s\r\n', + http_hlen)) + else + table.insert(respond_headers, 'Encapsulated: res-body=0\r\n') + end + + table.insert(respond_headers, '\r\n') + for _, h in ipairs(resp_req_headers) do + table.insert(respond_headers, h) + end + table.insert(respond_headers, '\r\n') + for _, h in ipairs(resp_http_headers) do + table.insert(respond_headers, h) + end + table.insert(respond_headers, '\r\n') + table.insert(respond_headers, chunked_size .. '\r\n') + table.insert(respond_headers, content) + table.insert(respond_headers, '\r\n0\r\n\r\n') + return respond_headers + end + + local function add_respond_header(name, value) + if name and value then + table.insert(respond_headers, string.format('%s: %s\r\n', name, value)) + end + end + + local function result_header_table(result) + local icap_headers = {} + for s in result:gmatch("[^\r\n]+") do + if string.find(s, '^ICAP') then + icap_headers['icap'] = tostring(s) + elseif string.find(s, '^HTTP') then + icap_headers['http'] = tostring(s) + elseif string.find(s, '[%a%d-+]-:') then + local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)") + if key ~= nil then + icap_headers[key:lower()] = tostring(value) + end + end + end + lua_util.debugm(rule.name, task, '%s: icap_headers: %s', + rule.log_prefix, icap_headers) + return icap_headers + end + + local function threat_table_add(icap_threat, maybe_split) + + if maybe_split and string.find(icap_threat, ',') then + local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {} + + for _, v in ipairs(threats) do + table.insert(threat_table, v) + end + else + table.insert(threat_table, icap_threat) + end + return true + end + + local function icap_parse_result(headers) + + --[[ + @ToDo: handle type in response + + Generic Strings: + icap: X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC; + icap: X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader; + + Symantec String: + icap: X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation + icap: X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation; + + Sophos Strings: + icap: X-Virus-ID: Troj/DocDl-OYC + http: X-Blocked: Virus found during virus scan + http: X-Blocked-By: Sophos Anti-Virus + + Kaspersky Web Traffic Security Strings: + icap: X-Virus-ID: HEUR:Backdoor.Java.QRat.gen + icap: X-Response-Info: blocked + icap: X-Virus-ID: no threats + icap: X-Response-Info: blocked + icap: X-Response-Info: passed + http: HTTP/1.1 403 Forbidden + + Kaspersky Scan Engine 2.0 (ICAP mode) + icap: X-Virus-ID: EICAR-Test-File + http: HTTP/1.0 403 Forbidden + + Trend Micro Strings: + icap: X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1 + icap: X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1; + http: HTTP/1.1 403 Forbidden (TMWS Blocked) + http: HTTP/1.1 403 Forbidden + + F-Secure Internet Gatekeeper Strings: + icap: X-FSecure-Scan-Result: infected + icap: X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203" + icap: X-FSecure-Infected-Filename: "virus.doc" + + ESET File Security for Linux 7.0 + icap: X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA; + icap: X-Virus-ID: Trojaner + icap: X-Response-Info: Blocked + + McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules) + icap: X-Virus-ID: EICAR test file + icap: X-Media-Type: text/plain + icap: X-Block-Result: 80 + icap: X-Block-Reason: Malware found + icap: X-Block-Reason: Archive not supported + icap: X-Block-Reason: Media Type (Block List) + http: HTTP/1.0 403 VirusFound + + C-ICAP Squidclamav + icap/http: X-Infection-Found: Type=0; Resolution=2; Threat={HEX}EICAR.TEST.3.UNOFFICIAL; + icap/http: X-Virus-ID: {HEX}EICAR.TEST.3.UNOFFICIAL + http: HTTP/1.0 307 Temporary Redirect + ]] -- + + -- Generic ICAP Headers + if headers['x-infection-found'] then + local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") + + -- Type=2 is typical for scan error returns + if icap_type and icap_type == '2' then + lua_util.debugm(rule.name, task, + '%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat) + common.yield_result(task, rule, icap_threat, 0, + 'fail', maybe_part) + return true + elseif icap_threat ~= nil then + lua_util.debugm(rule.name, task, + '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat) + threat_table_add(icap_threat, false) + -- stupid workaround for unuseable x-infection-found header + -- but also x-virus-name set (McAfee Web Gateway 9) + elseif not icap_threat and headers['x-virus-name'] then + threat_table_add(headers['x-virus-name'], true) + else + threat_table_add(headers['x-infection-found'], true) + end + elseif headers['x-virus-name'] and headers['x-virus-name'] ~= "no threats" then + lua_util.debugm(rule.name, task, + '%s: icap X-Virus-Name: %s', rule.log_prefix, headers['x-virus-name']) + threat_table_add(headers['x-virus-name'], true) + elseif headers['x-virus-id'] and headers['x-virus-id'] ~= "no threats" then + lua_util.debugm(rule.name, task, + '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id']) + threat_table_add(headers['x-virus-id'], true) + -- FSecure X-Headers + elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then + + local infected_filename = "" + local infection_name = "-unknown-" + + if headers['x-fsecure-infected-filename'] then + infected_filename = string.gsub(headers['x-fsecure-infected-filename'], '[%s"]', '') + end + if headers['x-fsecure-infection-name'] then + infection_name = string.gsub(headers['x-fsecure-infection-name'], '[%s"]', '') + end + + lua_util.debugm(rule.name, task, + '%s: icap X-FSecure-Infection-Name (X-FSecure-Infected-Filename): %s (%s)', + rule.log_prefix, infection_name, infected_filename) + + threat_table_add(infection_name, true) + -- McAfee Web Gateway manual extra headers + elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then + threat_table_add(headers['x-mwg-block-reason'], false) + -- Sophos SAVDI special http headers + elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then + threat_table_add(headers['x-blocked'], false) + elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then + threat_table_add(headers['x-block-reason'], false) + -- last try HTTP [4]xx return + elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then + threat_table_add( + string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + elseif rule.use_http_3xx_as_threat and + headers.http and + string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d') + then + threat_table_add( + string.format("pseudo-virus (redirect): %s", + string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + end + + if #threat_table > 0 then + common.yield_result(task, rule, threat_table, rule.default_score, nil, maybe_part) + common.save_cache(task, digest, rule, threat_table, rule.default_score, maybe_part) + return true + else + return false + end + end + + local function icap_r_respond_http_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_respond_http_cb") + else + local result = tostring(data) + + local icap_http_headers = result_header_table(result) or {} + -- Find HTTP/[12].x [234]xx response + if icap_http_headers.http and string.find(icap_http_headers.http, 'HTTP%/[12]%.. [234]%d%d') then + local icap_http_header_result = icap_parse_result(icap_http_headers) + if icap_http_header_result then + -- Threat found - close connection + connection:close() + else + common.save_cache(task, digest, rule, 'OK', 0, maybe_part) + common.log_clean(task, rule) + end + else + rspamd_logger.errx(task, '%s: unhandled response |%s|', + rule.log_prefix, string.gsub(result, "\r\n", ", ")) + common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap), + 0.0, 'fail', maybe_part) + end + end + end + + local function icap_r_respond_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_respond_cb") + else + local result = tostring(data) + + local icap_headers = result_header_table(result) or {} + -- Find ICAP/1.x 2xx response + if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then + local icap_header_result = icap_parse_result(icap_headers) + if icap_header_result then + -- Threat found - close connection + connection:close() + elseif not icap_header_result + and rule.use_http_result_header + and icap_headers.encapsulated + and not string.find(icap_headers.encapsulated, 'null%-body=0') + then + -- Try to read encapsulated HTTP Headers + lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers', + rule.log_prefix) + connection:add_read(icap_r_respond_http_cb, '\r\n\r\n') + else + connection:close() + common.save_cache(task, digest, rule, 'OK', 0, maybe_part) + common.log_clean(task, rule) + end + elseif icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then + -- Find ICAP/1.x 5/4xx response + --[[ + Symantec String: + ICAP/1.0 539 Aborted - No AV scanning license + SquidClamAV/C-ICAP: + ICAP/1.0 500 Server error + Eset: + ICAP/1.0 405 Forbidden + TrendMicro: + ICAP/1.0 400 Bad request + McAfee: + ICAP/1.0 418 Bad composition + ]]-- + rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap) + common.yield_result(task, rule, icap_headers.icap, 0.0, + 'fail', maybe_part) + return false + else + rspamd_logger.errx(task, '%s: unhandled response |%s|', + rule.log_prefix, string.gsub(result, "\r\n", ", ")) + common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap), + 0.0, 'fail', maybe_part) + end + end + end + + local function icap_w_respond_cb(err_m, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_w_respond_cb") + else + connection:add_read(icap_r_respond_cb, '\r\n\r\n') + end + end + + local function icap_r_options_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_options_cb") + else + local icap_headers = result_header_table(tostring(data)) + + if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then + if icap_headers['methods'] and string.find(icap_headers['methods'], 'RESPMOD') then + -- Allow "204 No Content" responses + -- https://datatracker.ietf.org/doc/html/rfc3507#section-4.6 + if icap_headers['allow'] and string.find(icap_headers['allow'], '204') then + add_respond_header('Allow', '204') + end + + if rule.x_client_header then + local client = task:get_from_ip() + if client then + add_respond_header('X-Client-IP', client:to_string()) + end + end + + -- F-Secure extra headers + if icap_headers['server'] and string.find(icap_headers['server'], 'f-secure icap server') then + + if rule.x_rcpt_header then + local rcpt_to = task:get_principal_recipient() + if rcpt_to then + add_respond_header('X-Rcpt-To', rcpt_to) + end + end + + if rule.x_from_header then + local mail_from = task:get_principal_recipient() + if mail_from and mail_from[1] then + add_respond_header('X-Rcpt-To', mail_from[1].addr) + end + end + + end + + if icap_headers.connection and icap_headers.connection:lower() == 'close' then + lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection', + rule.log_prefix, icap_headers.connection) + connection:close() + tcp_options.callback = icap_w_respond_cb + tcp_options.data = get_respond_query() + tcp.request(tcp_options) + else + connection:add_write(icap_w_respond_cb, get_respond_query()) + end + + else + rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s', + rule.log_prefix, icap_headers['methods']) + common.yield_result(task, rule, 'NO RESPMOD', 0.0, + 'fail', maybe_part) + end + else + rspamd_logger.errx(task, '%s: OPTIONS query failed: %s', + rule.log_prefix, icap_headers.icap or "-") + common.yield_result(task, rule, 'OPTIONS query failed', 0.0, + 'fail', maybe_part) + end + end + end + + if err or conn == nil then + icap_requery(err, "options_request") + else + conn:add_read(icap_r_options_cb, '\r\n\r\n') + end + end + + tcp_options.task = task + tcp_options.stop_pattern = '\r\n' + tcp_options.read = false + tcp_options.timeout = rule.timeout + tcp_options.callback = icap_callback + tcp_options.data = options_request + + if rule.ssl then + tcp_options.ssl = true + if rule.no_ssl_verify then + tcp_options.no_ssl_verify = true + end + end + + tcp_options.host = addr:to_string() + tcp_options.port = addr:get_port() + tcp_options.upstream = upstream + + tcp.request(tcp_options) + end + + if common.condition_check_and_continue(task, content, rule, digest, + icap_check_uncached, maybe_part) then + return + else + icap_check_uncached() + end + +end + +return { + type = { N, 'virus', 'virus', 'scanner' }, + description = 'generic icap antivirus', + configure = icap_config, + check = icap_check, + name = N +} diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua new file mode 100644 index 0000000..e47cebe --- /dev/null +++ b/lualib/lua_scanners/init.lua @@ -0,0 +1,75 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_scanners +-- This module contains external scanners functions +--]] + +local fun = require "fun" + +local exports = { +} + +local function require_scanner(name) + local sc = require("lua_scanners/" .. name) + + exports[sc.name or name] = sc +end + +-- Antiviruses +require_scanner('clamav') +require_scanner('fprot') +require_scanner('kaspersky_av') +require_scanner('kaspersky_se') +require_scanner('savapi') +require_scanner('sophos') +require_scanner('virustotal') +require_scanner('avast') + +-- Other scanners +require_scanner('dcc') +require_scanner('oletools') +require_scanner('icap') +require_scanner('vadesecure') +require_scanner('spamassassin') +require_scanner('p0f') +require_scanner('razor') +require_scanner('pyzor') +require_scanner('cloudmark') + +exports.add_scanner = function(name, t, conf_func, check_func) + assert(type(conf_func) == 'function' and type(check_func) == 'function', + 'bad arguments') + exports[name] = { + type = t, + configure = conf_func, + check = check_func, + } +end + +exports.filter = function(t) + return fun.tomap(fun.filter(function(_, elt) + return type(elt) == 'table' and elt.type and ( + (type(elt.type) == 'string' and elt.type == t) or + (type(elt.type) == 'table' and fun.any(function(tt) + return tt == t + end, elt.type)) + ) + end, exports)) +end + +return exports diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua new file mode 100644 index 0000000..d52cef0 --- /dev/null +++ b/lualib/lua_scanners/kaspersky_av.lua @@ -0,0 +1,197 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module kaspersky +-- This module contains kaspersky antivirus access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_util = require "rspamd_util" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "kaspersky" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function kaspersky_config(opts) + local kaspersky_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + product_id = 0, + log_clean = false, + timeout = 5.0, + retransmits = 1, -- use local files, retransmits are useless + cache_expire = 3600, -- expire redis in one hour + message = default_message, + detection_category = "virus", + tmpdir = '/tmp', + } + + kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts) + + if not kaspersky_conf.prefix then + kaspersky_conf.prefix = 'rs_' .. kaspersky_conf.name .. '_' + end + + if not kaspersky_conf.log_prefix then + if kaspersky_conf.name:lower() == kaspersky_conf.type:lower() then + kaspersky_conf.log_prefix = kaspersky_conf.name + else + kaspersky_conf.log_prefix = kaspersky_conf.name .. ' (' .. kaspersky_conf.type .. ')' + end + end + + if not kaspersky_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + kaspersky_conf['upstreams'] = upstream_list.create(rspamd_config, + kaspersky_conf['servers'], 0) + + if kaspersky_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', kaspersky_conf.name) + return kaspersky_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + kaspersky_conf['servers']) + return nil +end + +local function kaspersky_check(task, content, digest, rule, maybe_part) + local function kaspersky_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local fname = string.format('%s/%s.tmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + local clamav_compat_cmd = string.format("nSCAN %s\n", fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for kaspersky scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure file cleanup + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + local function kaspersky_callback(err, data) + if err then + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = kaspersky_callback, + data = { clamav_compat_cmd }, + stop_pattern = '\n' + }) + else + rspamd_logger.errx(task, + '%s [%s]: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + common.yield_result(task, rule, + 'failed to scan and retransmits exceed', 0.0, 'fail', + maybe_part) + end + + else + data = tostring(data) + local cached + lua_util.debugm(rule.name, task, + '%s [%s]: got reply: %s', + rule['symbol'], rule['type'], data) + if data == 'stream: OK' or data == fname .. ': OK' then + cached = 'OK' + common.log_clean(task, rule) + else + local vname = string.match(data, ': (.+) FOUND') + if vname then + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + cached = vname + else + rspamd_logger.errx(task, 'unhandled response: %s', data) + common.yield_result(task, rule, 'unhandled response', + 0.0, 'fail', maybe_part) + end + end + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = kaspersky_callback, + data = { clamav_compat_cmd }, + stop_pattern = '\n' + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, + kaspersky_check_uncached, maybe_part) then + return + else + kaspersky_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'kaspersky antivirus', + configure = kaspersky_config, + check = kaspersky_check, + name = N +} diff --git a/lualib/lua_scanners/kaspersky_se.lua b/lualib/lua_scanners/kaspersky_se.lua new file mode 100644 index 0000000..5e0f2ea --- /dev/null +++ b/lualib/lua_scanners/kaspersky_se.lua @@ -0,0 +1,287 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module kaspersky_se +-- This module contains Kaspersky Scan Engine integration support +-- https://www.kaspersky.com/scan-engine +--]] + +local lua_util = require "lua_util" +local rspamd_util = require "rspamd_util" +local http = require "rspamd_http" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = 'kaspersky_se' + +local function kaspersky_se_config(opts) + + local default_conf = { + name = N, + default_port = 9999, + use_https = false, + use_files = false, + timeout = 5.0, + log_clean = false, + tmpdir = '/tmp', + retransmits = 1, + cache_expire = 7200, -- expire redis in 2h + message = '${SCANNER}: spam message found: "${VIRUS}"', + detection_category = "virus", + default_score = 1, + action = false, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + } + + default_conf = lua_util.override_defaults(default_conf, opts) + + if not default_conf.prefix then + default_conf.prefix = 'rs_' .. default_conf.name .. '_' + end + + if not default_conf.log_prefix then + if default_conf.name:lower() == default_conf.type:lower() then + default_conf.log_prefix = default_conf.name + else + default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')' + end + end + + if not default_conf.servers and default_conf.socket then + default_conf.servers = default_conf.socket + end + + if not default_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + default_conf.upstreams = upstream_list.create(rspamd_config, + default_conf.servers, + default_conf.default_port) + + if default_conf.upstreams then + lua_util.add_debug_alias('external_services', default_conf.name) + return default_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + default_conf['servers']) + return nil +end + +local function kaspersky_se_check(task, content, digest, rule, maybe_part) + local function kaspersky_se_check_uncached() + local function make_url(addr) + local url + local suffix = '/scanmemory' + + if rule.use_files then + suffix = '/scanfile' + end + if rule.use_https then + url = string.format('https://%s:%d%s', tostring(addr), + addr:get_port(), suffix) + else + url = string.format('http://%s:%d%s', tostring(addr), + addr:get_port(), suffix) + end + + return url + end + + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + local url = make_url(addr) + local hdrs = { + ['X-KAV-ProtocolVersion'] = '1', + ['X-KAV-Timeout'] = tostring(rule.timeout * 1000), + } + + if task:has_from() then + hdrs['X-KAV-ObjectURL'] = string.format('[from:%s]', task:get_from()[1].addr) + end + + local req_body + + if rule.use_files then + local fname = string.format('%s/%s.tmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for kaspersky_se scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure cleanup + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + req_body = fname + else + req_body = content + end + + local request_data = { + task = task, + url = url, + body = req_body, + headers = hdrs, + timeout = rule.timeout, + } + + local function kas_callback(http_err, code, body, headers) + + local function requery() + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, + '%s: Request Error: %s - retries left: %s', + rule.log_prefix, http_err, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + url = make_url(addr) + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + request_data.url = url + request_data.upstream = upstream + + http.request(request_data) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. + 'retransmits exceed') + end + end + + if http_err then + requery() + else + -- Parse the response + if upstream then + upstream:ok() + end + if code ~= 200 then + rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) + return + end + local data = string.gsub(tostring(body), '[\r\n%s]$', '') + local cached + lua_util.debugm(rule.name, task, '%s: got reply data: "%s"', + rule.log_prefix, data) + + if data:find('^CLEAN') then + -- Handle CLEAN replies + if data == 'CLEAN' then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: message or mime_part is clean', + rule.log_prefix) + else + lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', + rule.log_prefix) + end + elseif data == 'CLEAN AND CONTAINS OFFICE MACRO' then + common.yield_result(task, rule, 'File contains macros', + 0.0, 'macro', maybe_part) + cached = 'MACRO' + else + rspamd_logger.errx(task, '%s: unhandled clean response: %s', rule.log_prefix, data) + common.yield_result(task, rule, 'unhandled response:' .. data, + 0.0, 'fail', maybe_part) + end + elseif data == 'SERVER_ERROR' then + rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, data) + common.yield_result(task, rule, 'error:' .. data, + 0.0, 'fail', maybe_part) + elseif string.match(data, 'DETECT (.+)') then + local vname = string.match(data, 'DETECT (.+)') + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + cached = vname + elseif string.match(data, 'NON_SCANNED %((.+)%)') then + local why = string.match(data, 'NON_SCANNED %((.+)%)') + + if why == 'PASSWORD PROTECTED' then + rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) + common.yield_result(task, rule, 'File is encrypted: ' .. why, + 0.0, 'encrypted', maybe_part) + cached = 'ENCRYPTED' + else + common.yield_result(task, rule, 'unhandled response:' .. data, + 0.0, 'fail', maybe_part) + end + else + rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data) + common.yield_result(task, rule, 'unhandled response:' .. data, + 0.0, 'fail', maybe_part) + end + + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + end + + end + end + + request_data.callback = kas_callback + http.request(request_data) + end + + if common.condition_check_and_continue(task, content, rule, digest, + kaspersky_se_check_uncached, maybe_part) then + return + else + + kaspersky_se_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'Kaspersky Scan Engine interface', + configure = kaspersky_se_config, + check = kaspersky_se_check, + name = N +} diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua new file mode 100644 index 0000000..378e094 --- /dev/null +++ b/lualib/lua_scanners/oletools.lua @@ -0,0 +1,369 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module oletools +-- This module contains oletools access functions. +-- Olefy is needed: https://github.com/HeinleinSupport/olefy +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local ucl = require "ucl" +local common = require "lua_scanners/common" + +local N = 'oletools' + +local function oletools_config(opts) + + local oletools_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + default_port = 10050, + timeout = 15.0, + log_clean = false, + retransmits = 2, + cache_expire = 86400, -- expire redis in 1d + min_size = 500, + symbol = "OLETOOLS", + message = '${SCANNER}: Oletools threat message found: "${VIRUS}"', + detection_category = "office macro", + default_score = 1, + action = false, + extended = false, + symbol_type = 'postfilter', + dynamic_scan = true, + } + + oletools_conf = lua_util.override_defaults(oletools_conf, opts) + + if not oletools_conf.prefix then + oletools_conf.prefix = 'rs_' .. oletools_conf.name .. '_' + end + + if not oletools_conf.log_prefix then + if oletools_conf.name:lower() == oletools_conf.type:lower() then + oletools_conf.log_prefix = oletools_conf.name + else + oletools_conf.log_prefix = oletools_conf.name .. ' (' .. oletools_conf.type .. ')' + end + end + + if not oletools_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + oletools_conf.upstreams = upstream_list.create(rspamd_config, + oletools_conf.servers, + oletools_conf.default_port) + + if oletools_conf.upstreams then + lua_util.add_debug_alias('external_services', oletools_conf.name) + return oletools_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + oletools_conf.servers) + return nil +end + +local function oletools_check(task, content, digest, rule, maybe_part) + local function oletools_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n' + local json_response = "" + + local function oletools_callback(err, data, conn) + + local function oletools_requery(error) + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout, + shutdown = true, + data = { protocol, content }, + callback = oletools_callback, + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - err: %s', rule.log_prefix, error) + common.yield_result(task, rule, + 'failed to scan, maximum retransmits exceed - err: ' .. error, + 0.0, 'fail', maybe_part) + end + end + + if err then + + oletools_requery(err) + + else + json_response = json_response .. tostring(data) + + if not string.find(json_response, '\t\n\n\t') and #data == 8192 then + lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s', + rule.log_prefix, #json_response, #data) + conn:add_read(oletools_callback) + + else + local ucl_parser = ucl.parser() + local ok, ucl_err = ucl_parser:parse_string(tostring(json_response)) + if not ok then + rspamd_logger.errx(task, "%s: error parsing json response, retry: %s", + rule.log_prefix, ucl_err) + oletools_requery(ucl_err) + return + end + + local result = ucl_parser:get_object() + + local oletools_rc = { + [0] = 'RETURN_OK', + [1] = 'RETURN_WARNINGS', + [2] = 'RETURN_WRONG_ARGS', + [3] = 'RETURN_FILE_NOT_FOUND', + [4] = 'RETURN_XGLOB_ERR', + [5] = 'RETURN_OPEN_ERROR', + [6] = 'RETURN_PARSE_ERROR', + [7] = 'RETURN_SEVERAL_ERRS', + [8] = 'RETURN_UNEXPECTED', + [9] = 'RETURN_ENCRYPTED', + } + + -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs, + -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings + -- Keep sorted to avoid dragons + local analysis_cat_table = { + autoexec = '-', + base64 = '-', + dridex = '-', + hex = '-', + iocs = '-', + macro_exist = '-', + suspicious = '-', + vba = '-' + } + local analysis_keyword_table = {} + + for _, v in ipairs(result) do + + if v.error ~= nil and v.type ~= 'error' then + -- olefy, not oletools error + rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix, + v.error) + if v.error == 'File too small' then + common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part) + common.log_clean(task, rule, 'File too small to be scanned for macros') + return + else + oletools_requery(v.error) + end + + elseif tostring(v.type) == "MetaInformation" and v.version ~= nil then + -- if MetaInformation section - check and print script and version + + lua_util.debugm(N, task, '%s: version: %s %s', rule.log_prefix, + tostring(v.script_name), tostring(v.version)) + + elseif tostring(v.type) == "MetaInformation" and v.return_code ~= nil then + -- if MetaInformation section - check return_code + + local oletools_rc_code = tonumber(v.return_code) + if oletools_rc_code == 9 then + rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix) + common.yield_result(task, rule, + 'failed - err: ' .. oletools_rc[oletools_rc_code], + 0.0, 'encrypted', maybe_part) + common.save_cache(task, digest, rule, 'encrypted', 1.0, maybe_part) + return + elseif oletools_rc_code == 5 then + rspamd_logger.warnx(task, '%s: olefy could not open the file - error: %s', rule.log_prefix, + result[2]['message']) + common.yield_result(task, rule, + 'failed - err: ' .. oletools_rc[oletools_rc_code], + 0.0, 'fail', maybe_part) + return + elseif oletools_rc_code > 6 then + rspamd_logger.errx(task, '%s: MetaInfo section error code: %s', + rule.log_prefix, oletools_rc[oletools_rc_code]) + rspamd_logger.errx(task, '%s: MetaInfo section message: %s', + rule.log_prefix, result[2]['message']) + common.yield_result(task, rule, + 'failed - err: ' .. oletools_rc[oletools_rc_code], + 0.0, 'fail', maybe_part) + return + elseif oletools_rc_code > 1 then + rspamd_logger.errx(task, '%s: Error message: %s', + rule.log_prefix, result[2]['message']) + oletools_requery(oletools_rc[oletools_rc_code]) + end + + elseif tostring(v.type) == "error" then + -- error section found - check message + rspamd_logger.errx(task, '%s: Error section error code: %s', + rule.log_prefix, v.error) + rspamd_logger.errx(task, '%s: Error section message: %s', + rule.log_prefix, v.message) + --common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail') + + elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then + -- analysis + macro found - evaluate response + + if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then + rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix) + oletools_requery('oletools unhandled error') + + elseif #v.macros > 0 then + + analysis_cat_table.macro_exist = 'M' + + lua_util.debugm(rule.name, task, + '%s: filename: %s', rule.log_prefix, result[2]['file']) + lua_util.debugm(rule.name, task, + '%s: type: %s', rule.log_prefix, result[2]['type']) + + for _, m in ipairs(v.macros) do + lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' .. + 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename) + end + + for _, a in ipairs(v.analysis) do + lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' .. + 'description: %s', rule.log_prefix, a.type, a.keyword, a.description) + if a.type == 'AutoExec' then + analysis_cat_table.autoexec = 'A' + table.insert(analysis_keyword_table, a.keyword) + elseif a.type == 'Suspicious' then + if rule.extended == true or + (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings') + then + analysis_cat_table.suspicious = 'S' + table.insert(analysis_keyword_table, a.keyword) + end + elseif a.type == 'IOC' then + analysis_cat_table.iocs = 'I' + elseif a.type == 'Hex strings' then + analysis_cat_table.hex = 'H' + elseif a.type == 'Base64 strings' then + analysis_cat_table.base64 = 'B' + elseif a.type == 'Dridex strings' then + analysis_cat_table.dridex = 'D' + elseif a.type == 'VBA strings' then + analysis_cat_table.vba = 'V' + end + end + end + end + end + + lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table) + lua_util.debugm(N, task, '%s: analysis_cat_table: %s', rule.log_prefix, analysis_cat_table) + + if rule.extended == false and analysis_cat_table.autoexec == 'A' and analysis_cat_table.suspicious == 'S' then + -- use single string as virus name + local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')' + lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat) + common.yield_result(task, rule, threat, rule.default_score, nil, maybe_part) + common.save_cache(task, digest, rule, threat, rule.default_score, maybe_part) + + elseif rule.extended == true and #analysis_keyword_table > 0 then + -- report any flags (types) and any most keywords as individual virus name + local analysis_cat_table_values_sorted = {} + + -- see https://github.com/rspamd/rspamd/commit/6bd3e2b9f49d1de3ab882aeca9c30bc7d526ac9d#commitcomment-40130493 + -- for details + local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table) + table.sort(analysis_cat_table_keys_sorted) + + for _, v in ipairs(analysis_cat_table_keys_sorted) do + table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v]) + end + + table.insert(analysis_keyword_table, 1, table.concat(analysis_cat_table_values_sorted)) + + lua_util.debugm(rule.name, task, '%s: extended threat result: %s', + rule.log_prefix, table.concat(analysis_keyword_table, ',')) + + common.yield_result(task, rule, analysis_keyword_table, + rule.default_score, nil, maybe_part) + common.save_cache(task, digest, rule, analysis_keyword_table, + rule.default_score, maybe_part) + + elseif analysis_cat_table.macro_exist == '-' and #analysis_keyword_table == 0 then + common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part) + common.log_clean(task, rule, 'No macro found') + + else + common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part) + common.log_clean(task, rule, 'Scanned Macro is OK') + end + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout, + shutdown = true, + data = { protocol, content }, + callback = oletools_callback, + }) + + end + + if common.condition_check_and_continue(task, content, rule, digest, + oletools_check_uncached, maybe_part) then + return + else + oletools_check_uncached() + end + +end + +return { + type = { N, 'attachment scanner', 'hash', 'scanner' }, + description = 'oletools office macro scanner', + configure = oletools_config, + check = oletools_check, + name = N +} diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua new file mode 100644 index 0000000..7785f83 --- /dev/null +++ b/lualib/lua_scanners/p0f.lua @@ -0,0 +1,227 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module p0f +-- This module contains p0f access functions +--]] + +local tcp = require "rspamd_tcp" +local rspamd_util = require "rspamd_util" +local rspamd_logger = require "rspamd_logger" +local lua_redis = require "lua_redis" +local lua_util = require "lua_util" +local common = require "lua_scanners/common" + +-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317 +local S = { + BAD_QUERY = 0x0, + OK = 0x10, + NO_MATCH = 0x20 +} + +local N = 'p0f' + +local function p0f_check(task, ip, rule) + + local function ip2bin(addr) + addr = addr:to_table() + + for k, v in ipairs(addr) do + addr[k] = rspamd_util.pack('B', v) + end + + return table.concat(addr) + end + + local function trim(...) + local vars = { ... } + + for k, v in ipairs(vars) do + -- skip numbers, trim only strings + if tonumber(vars[k]) == nil then + vars[k] = string.gsub(v, '[^%w-_\\.\\(\\) ]', '') + end + end + + return lua_util.unpack(vars) + end + + local function parse_p0f_response(data) + --[[ + p0f_api_response[232]: magic, status, first_seen, last_seen, total_conn, + uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q, + os_name, os_flavor, http_name, http_flavor, link_type, language + ]]-- + + data = tostring(data) + + -- API response must be 232 bytes long + if #data ~= 232 then + rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes', + rule.socket, #data) + + common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket, + 0.0, 'fail') + return + end + + local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name, + os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( + 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data)) + + if status ~= S.OK then + if status == S.BAD_QUERY then + rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket) + common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket, + 0.0, 'fail') + end + + return + end + + local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor + + task:get_mempool():set_variable('os_fingerprint', os_string, link_type, + uptime_min, distance) + + if link_type and #link_type > 0 then + common.yield_result(task, rule, { + os_string, + 'link=' .. link_type, + 'distance=' .. distance }, + 0.0) + else + common.yield_result(task, rule, { + os_string, + 'link=unknown', + 'distance=' .. distance }, + 0.0) + end + + return data + end + + local function make_p0f_request() + + local function check_p0f_cb(err, data) + + local function redis_set_cb(redis_set_err) + if redis_set_err then + rspamd_logger.errx(task, 'redis received an error: %s', redis_set_err) + end + end + + if err then + rspamd_logger.errx(task, 'p0f received an error: %s', err) + common.yield_result(task, rule, 'Error getting result: ' .. err, + 0.0, 'fail') + return + end + + data = parse_p0f_response(data) + + if rule.redis_params and data then + local key = rule.prefix .. ip:to_string() + local ret = lua_redis.redis_make_request(task, + rule.redis_params, + key, + true, + redis_set_cb, + 'SETEX', + { key, tostring(rule.expire), data } + ) + + if not ret then + rspamd_logger.warnx(task, 'error connecting to redis') + end + end + end + + local query = rspamd_util.pack('I4 I1 c16', 0x50304601, + ip:get_version(), ip2bin(ip)) + + tcp.request({ + host = rule.socket, + callback = check_p0f_cb, + data = { query }, + task = task, + timeout = rule.timeout + }) + end + + local function redis_get_cb(err, data) + if err or type(data) ~= 'string' then + make_p0f_request() + else + parse_p0f_response(data) + end + end + + local ret = nil + if rule.redis_params then + local key = rule.prefix .. ip:to_string() + ret = lua_redis.redis_make_request(task, + rule.redis_params, + key, + false, + redis_get_cb, + 'GET', + { key } + ) + end + + if not ret then + make_p0f_request() -- fallback to directly querying p0f + end +end + +local function p0f_config(opts) + local p0f_conf = { + name = N, + timeout = 5, + symbol = 'P0F', + symbol_fail = 'P0F_FAIL', + patterns = {}, + expire = 7200, + prefix = 'p0f', + detection_category = 'fingerprint', + message = '${SCANNER}: fingerprint matched: "${VIRUS}"' + } + + p0f_conf = lua_util.override_defaults(p0f_conf, opts) + p0f_conf.patterns = common.create_regex_table(p0f_conf.patterns) + + if not p0f_conf.log_prefix then + p0f_conf.log_prefix = p0f_conf.name + end + + if not p0f_conf.socket then + rspamd_logger.errx(rspamd_config, 'no servers defined') + return nil + end + + return p0f_conf +end + +return { + type = { N, 'fingerprint', 'scanner' }, + description = 'passive OS fingerprinter', + configure = p0f_config, + check = p0f_check, + name = N +} diff --git a/lualib/lua_scanners/pyzor.lua b/lualib/lua_scanners/pyzor.lua new file mode 100644 index 0000000..75c1b4a --- /dev/null +++ b/lualib/lua_scanners/pyzor.lua @@ -0,0 +1,206 @@ +--[[ +Copyright (c) 2021, defkev <defkev@gmail.com> +Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de> +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module pyzor +-- This module contains pyzor access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = 'pyzor' +local categories = { 'pyzor', 'bulk', 'hash', 'scanner' } + +local function pyzor_config(opts) + + local pyzor_conf = { + text_part_min_words = 2, + default_port = 5953, + timeout = 15.0, + log_clean = false, + retransmits = 2, + detection_category = "hash", + cache_expire = 7200, -- expire redis in one hour + message = '${SCANNER}: Pyzor bulk message found: "${VIRUS}"', + default_score = 1.5, + action = false, + } + + pyzor_conf = lua_util.override_defaults(pyzor_conf, opts) + + if not pyzor_conf.prefix then + pyzor_conf.prefix = 'rext_' .. N .. '_' + end + + if not pyzor_conf.log_prefix then + pyzor_conf.log_prefix = N .. ' (' .. pyzor_conf.detection_category .. ')' + end + + if not pyzor_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + pyzor_conf['upstreams'] = upstream_list.create(rspamd_config, + pyzor_conf['servers'], + pyzor_conf.default_port) + + if pyzor_conf['upstreams'] then + lua_util.add_debug_alias('external_services', N) + return pyzor_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + pyzor_conf['servers']) + return nil +end + +local function pyzor_check(task, content, digest, rule) + local function pyzor_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + local function pyzor_callback(err, data, conn) + + if err then + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s: retry IP: %s:%s err: %s', + rule.log_prefix, addr, addr:get_port(), err) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + shutdown = true, + data = content, + callback = pyzor_callback, + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, + 'failed to scan and retransmits exceed') + end + else + -- pyzor output is unicode (\x09 -> tab, \0a -> newline) + -- public.pyzor.org:24441 (200, 'OK') 21285091 206759 + -- server:port Code Diag Count WL-Count + local str_data = tostring(data) + lua_util.debugm(N, task, '%s: returned data: %s', + rule.log_prefix, str_data) + -- If pyzor would return JSON this wouldn't be necessary + local resp = {} + for v in string.gmatch(str_data, '[^\t]+') do + table.insert(resp, v) + end + -- rspamd_logger.infox(task, 'resp: %s', resp) + if resp[2] ~= [[(200, 'OK')]] then + rspamd_logger.errx(task, "error parsing response: %s", str_data) + return + end + + local whitelisted = tonumber(resp[4]) + local reported = tonumber(resp[3]) + + --rspamd_logger.infox(task, "%s - count=%s wl=%s", addr:to_string(), reported, whitelisted) + + --[[ + Weight is Count - WL-Count of rule.default_score in percent, e.g. + SPAM: + Count: 100 (100%) + WL-Count: 1 (1%) + rule.default_score: 1 + Weight: 0.99 + HAM: + Count: 10 (100%) + WL-Count: 10 (100%) + rule.default_score: 1 + Weight: 0 + ]] + local weight = tonumber(string.format("%.2f", + rule.default_score * (reported - whitelisted) / (reported + whitelisted))) + local info = string.format("count=%d wl=%d", reported, whitelisted) + local threat_string = string.format("bl_%d_wl_%d", + reported, whitelisted) + + if weight > 0 then + lua_util.debugm(N, task, '%s: returned result is spam - info: %s', + rule.log_prefix, info) + common.yield_result(task, rule, threat_string, weight) + common.save_cache(task, digest, rule, threat_string, weight) + else + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result is ham - info: %s', + rule.log_prefix, info) + else + lua_util.debugm(N, task, '%s: returned result is ham - info: %s', + rule.log_prefix, info) + end + common.save_cache(task, digest, rule, 'OK', weight) + end + + end + end + + if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then + rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) + return + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout, + shutdown = true, + data = content, + callback = pyzor_callback, + }) + end + if common.condition_check_and_continue(task, content, rule, digest, pyzor_check_uncached) then + return + else + pyzor_check_uncached() + end +end + +return { + type = categories, + description = 'pyzor bulk scanner', + configure = pyzor_config, + check = pyzor_check, + name = N +} diff --git a/lualib/lua_scanners/razor.lua b/lualib/lua_scanners/razor.lua new file mode 100644 index 0000000..fcc0a8e --- /dev/null +++ b/lualib/lua_scanners/razor.lua @@ -0,0 +1,181 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module razor +-- This module contains razor access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = 'razor' + +local function razor_config(opts) + + local razor_conf = { + name = N, + default_port = 11342, + timeout = 5.0, + log_clean = false, + retransmits = 2, + cache_expire = 7200, -- expire redis in 2h + message = '${SCANNER}: spam message found: "${VIRUS}"', + detection_category = "hash", + default_score = 1, + action = false, + dynamic_scan = false, + symbol_fail = 'RAZOR_FAIL', + symbol = 'RAZOR', + } + + razor_conf = lua_util.override_defaults(razor_conf, opts) + + if not razor_conf.prefix then + razor_conf.prefix = 'rs_' .. razor_conf.name .. '_' + end + + if not razor_conf.log_prefix then + razor_conf.log_prefix = razor_conf.name + end + + if not razor_conf.servers and razor_conf.socket then + razor_conf.servers = razor_conf.socket + end + + if not razor_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + razor_conf.upstreams = upstream_list.create(rspamd_config, + razor_conf.servers, + razor_conf.default_port) + + if razor_conf.upstreams then + lua_util.add_debug_alias('external_services', razor_conf.name) + return razor_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + razor_conf['servers']) + return nil +end + +local function razor_check(task, content, digest, rule) + local function razor_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + local function razor_callback(err, data, conn) + + local function razor_requery() + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s', + rule.log_prefix, err, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout or 2.0, + shutdown = true, + data = content, + callback = razor_callback, + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') + end + end + + if err then + + razor_requery() + + else + --[[ + @todo: Razorsocket currently only returns ham or spam. When the wrapper is fixed we should add dynamic scores here. + Maybe check spamassassin implementation. + + This implementation is based on https://github.com/cgt/rspamd-plugins + Thanks @cgt! + ]] -- + + local threat_string = tostring(data) + if threat_string == "spam" then + lua_util.debugm(N, task, '%s: returned result is spam', rule['symbol'], rule['type']) + common.yield_result(task, rule, threat_string, rule.default_score) + common.save_cache(task, digest, rule, threat_string, rule.default_score) + elseif threat_string == "ham" then + if rule.log_clean then + rspamd_logger.infox(task, '%s: returned result is ham', rule['symbol'], rule['type']) + else + lua_util.debugm(N, task, '%s: returned result is ham', rule['symbol'], rule['type']) + end + common.save_cache(task, digest, rule, 'OK', rule.default_score) + else + rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string) + end + + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule.timeout or 2.0, + shutdown = true, + data = content, + callback = razor_callback, + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, razor_check_uncached) then + return + else + razor_check_uncached() + end +end + +return { + type = { 'razor', 'spam', 'hash', 'scanner' }, + description = 'razor bulk scanner', + configure = razor_config, + check = razor_check, + name = N +} diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua new file mode 100644 index 0000000..08f7b66 --- /dev/null +++ b/lualib/lua_scanners/savapi.lua @@ -0,0 +1,261 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module savapi +-- This module contains avira savapi antivirus access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_util = require "rspamd_util" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "savapi" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function savapi_config(opts) + local savapi_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + default_port = 4444, -- note: You must set ListenAddress in savapi.conf + product_id = 0, + log_clean = false, + timeout = 15.0, -- FIXME: this will break task_timeout! + retransmits = 1, -- FIXME: useless, for local files + cache_expire = 3600, -- expire redis in one hour + message = default_message, + detection_category = "virus", + tmpdir = '/tmp', + } + + savapi_conf = lua_util.override_defaults(savapi_conf, opts) + + if not savapi_conf.prefix then + savapi_conf.prefix = 'rs_' .. savapi_conf.name .. '_' + end + + if not savapi_conf.log_prefix then + if savapi_conf.name:lower() == savapi_conf.type:lower() then + savapi_conf.log_prefix = savapi_conf.name + else + savapi_conf.log_prefix = savapi_conf.name .. ' (' .. savapi_conf.type .. ')' + end + end + + if not savapi_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + savapi_conf['upstreams'] = upstream_list.create(rspamd_config, + savapi_conf['servers'], + savapi_conf.default_port) + + if savapi_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', savapi_conf.name) + return savapi_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + savapi_conf['servers']) + return nil +end + +local function savapi_check(task, content, digest, rule) + local function savapi_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local fname = string.format('%s/%s.tmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for savapi scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure cleanup + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + local vnames = {} + + -- Forward declaration for recursive calls + local savapi_scan1_cb + + local function savapi_fin_cb(err, conn) + local vnames_reordered = {} + -- Swap table + for virus, _ in pairs(vnames) do + table.insert(vnames_reordered, virus) + end + lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) + if #vnames_reordered > 0 then + local vname = {} + for _, virus in ipairs(vnames_reordered) do + table.insert(vname, virus) + end + + common.yield_result(task, rule, vname) + common.save_cache(task, digest, rule, vname) + end + if conn then + conn:close() + end + end + + local function savapi_scan2_cb(err, data, conn) + local result = tostring(data) + lua_util.debugm(rule.name, task, "%s: got reply: %s", + rule.type, result) + + -- Terminal response - clean + if string.find(result, '200') or string.find(result, '210') then + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type']) + end + common.save_cache(task, digest, rule, 'OK') + conn:add_write(savapi_fin_cb, 'QUIT\n') + + -- Terminal response - infected + elseif string.find(result, '319') then + conn:add_write(savapi_fin_cb, 'QUIT\n') + + -- Non-terminal response + elseif string.find(result, '310') then + local virus + virus = result:match "310.*<<<%s(.*)%s+;.*;.*" + if not virus then + virus = result:match "310%s(.*)%s+;.*;.*" + if not virus then + rspamd_logger.errx(task, "%s: virus result unparseable: %s", + rule['type'], result) + common.yield_result(task, rule, 'virus result unparseable: ' .. result, 0.0, 'fail') + return + end + end + -- Store unique virus names + vnames[virus] = 1 + -- More content is expected + conn:add_write(savapi_scan1_cb, '\n') + end + end + + savapi_scan1_cb = function(err, conn) + conn:add_read(savapi_scan2_cb, '\n') + end + + -- 100 PRODUCT:xyz + local function savapi_greet2_cb(err, data, conn) + local result = tostring(data) + if string.find(result, '100 PRODUCT') then + lua_util.debugm(rule.name, task, "%s: scanning file: %s", + rule['type'], fname) + conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n', + fname) }) + else + rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'], + rule['product_id']) + common.yield_result(task, rule, 'invalid product id: ' .. result, 0.0, 'fail') + conn:add_write(savapi_fin_cb, 'QUIT\n') + end + end + + local function savapi_greet1_cb(err, conn) + conn:add_read(savapi_greet2_cb, '\n') + end + + local function savapi_callback_init(err, data, conn) + if err then + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = savapi_callback_init, + stop_pattern = { '\n' }, + }) + else + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') + end + else + local result = tostring(data) + + -- 100 SAVAPI:4.0 greeting + if string.find(result, '100') then + conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) }) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = savapi_callback_init, + stop_pattern = { '\n' }, + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, savapi_check_uncached) then + return + else + savapi_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'savapi avira antivirus', + configure = savapi_config, + check = savapi_check, + name = N +} diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua new file mode 100644 index 0000000..d9b64f1 --- /dev/null +++ b/lualib/lua_scanners/sophos.lua @@ -0,0 +1,192 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module savapi +-- This module contains avira savapi antivirus access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = "sophos" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function sophos_config(opts) + local sophos_conf = { + name = N, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + default_port = 4010, + timeout = 15.0, + log_clean = false, + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + detection_category = "virus", + } + + sophos_conf = lua_util.override_defaults(sophos_conf, opts) + + if not sophos_conf.prefix then + sophos_conf.prefix = 'rs_' .. sophos_conf.name .. '_' + end + + if not sophos_conf.log_prefix then + if sophos_conf.name:lower() == sophos_conf.type:lower() then + sophos_conf.log_prefix = sophos_conf.name + else + sophos_conf.log_prefix = sophos_conf.name .. ' (' .. sophos_conf.type .. ')' + end + end + + if not sophos_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + sophos_conf['upstreams'] = upstream_list.create(rspamd_config, + sophos_conf['servers'], + sophos_conf.default_port) + + if sophos_conf['upstreams'] then + lua_util.add_debug_alias('antivirus', sophos_conf.name) + return sophos_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + sophos_conf['servers']) + return nil +end + +local function sophos_check(task, content, digest, rule, maybe_part) + local function sophos_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local protocol = 'SSSP/1.0\n' + local streamsize = string.format('SCANDATA %d\n', #content) + local bye = 'BYE\n' + + local function sophos_callback(err, data, conn) + + if err then + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s', + rule.log_prefix, err, addr, retransmits) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = sophos_callback, + data = { protocol, streamsize, content, bye } + }) + else + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + common.yield_result(task, rule, 'failed to scan and retransmits exceed', + 0.0, 'fail', maybe_part) + end + else + data = tostring(data) + lua_util.debugm(rule.name, task, + '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) + local vname = string.match(data, 'VIRUS (%S+) ') + local cached + if vname then + common.yield_result(task, rule, vname, 1.0, nil, maybe_part) + common.save_cache(task, digest, rule, vname, 1.0, maybe_part) + else + if string.find(data, 'DONE OK') then + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix) + else + lua_util.debugm(rule.name, task, + '%s: message or mime_part is clean', rule.log_prefix) + end + cached = 'OK' + -- not finished - continue + elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then + conn:add_read(sophos_callback) + elseif string.find(data, 'FAIL 0212') then + rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data) + common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)', + 0.0, 'encrypted', maybe_part) + cached = 'ENCRYPTED' + elseif string.find(data, 'REJ 4') then + rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data) + common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', + 0.0, 'fail', maybe_part) + -- explicitly set REJ1 message when SAVDIreports a protocol error + elseif string.find(data, 'REJ 1') then + rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) + common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', + 0.0, 'fail', maybe_part) + else + rspamd_logger.errx(task, 'unhandled response: %s', data) + common.yield_result(task, rule, 'unhandled response: ' .. data, + 0.0, 'fail', maybe_part) + end + if cached then + common.save_cache(task, digest, rule, cached, 1.0, maybe_part) + end + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + callback = sophos_callback, + data = { protocol, streamsize, content, bye } + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, + sophos_check_uncached, maybe_part) then + return + else + sophos_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'sophos antivirus', + configure = sophos_config, + check = sophos_check, + name = N +} diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua new file mode 100644 index 0000000..f425924 --- /dev/null +++ b/lualib/lua_scanners/spamassassin.lua @@ -0,0 +1,213 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module spamassassin +-- This module contains spamd access functions. +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = 'spamassassin' + +local function spamassassin_config(opts) + + local spamassassin_conf = { + N = N, + scan_mime_parts = false, + scan_text_mime = false, + scan_image_mime = false, + default_port = 783, + timeout = 15.0, + log_clean = false, + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + symbol = "SPAMD", + message = '${SCANNER}: Spamassassin bulk message found: "${VIRUS}"', + detection_category = "spam", + default_score = 1, + action = false, + extended = false, + symbol_type = 'postfilter', + dynamic_scan = true, + } + + spamassassin_conf = lua_util.override_defaults(spamassassin_conf, opts) + + if not spamassassin_conf.prefix then + spamassassin_conf.prefix = 'rs_' .. spamassassin_conf.name .. '_' + end + + if not spamassassin_conf.log_prefix then + if spamassassin_conf.name:lower() == spamassassin_conf.type:lower() then + spamassassin_conf.log_prefix = spamassassin_conf.name + else + spamassassin_conf.log_prefix = spamassassin_conf.name .. ' (' .. spamassassin_conf.type .. ')' + end + end + + if not spamassassin_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + spamassassin_conf.upstreams = upstream_list.create(rspamd_config, + spamassassin_conf.servers, + spamassassin_conf.default_port) + + if spamassassin_conf.upstreams then + lua_util.add_debug_alias('external_services', spamassassin_conf.N) + return spamassassin_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + spamassassin_conf.servers) + return nil +end + +local function spamassassin_check(task, content, digest, rule) + local function spamassassin_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + -- Build the spamd query + -- https://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL + local request_data = { + "HEADERS SPAMC/1.5\r\n", + "User: root\r\n", + "Content-length: " .. #content .. "\r\n", + "\r\n", + content, + } + + local function spamassassin_callback(err, data) + + local function spamassassin_requery(error) + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s', + rule.log_prefix, error, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + data = request_data, + callback = spamassassin_callback, + }) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - err: %s', rule.log_prefix, error) + common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail') + end + end + + if err then + + spamassassin_requery(err) + + else + --lua_util.debugm(rule.N, task, '%s: returned result: %s', rule.log_prefix, data) + + --[[ + patterns tested against Spamassassin 3.4.6 + + X-Spam-Status: No, score=1.1 required=5.0 tests=HTML_MESSAGE,MIME_HTML_ONLY, + TVD_RCVD_SPACE_BRACKET,UNPARSEABLE_RELAY autolearn=no + autolearn_force=no version=3.4.6 + ]] -- + local header = string.gsub(tostring(data), "[\r\n]+[\t ]", " ") + --lua_util.debugm(rule.N, task, '%s: returned header: %s', rule.log_prefix, header) + + local symbols = "" + local spam_score = 0 + for s in header:gmatch("[^\r\n]+") do + if string.find(s, 'X%-Spam%-Status: %S+, score') then + local pattern_symbols = "X%-Spam%-Status: %S+, score%=([%-%d%.]+)%s.*tests%=(.*,?)(%s*%S+)%sautolearn.*" + spam_score = string.gsub(s, pattern_symbols, "%1") + symbols = string.gsub(s, pattern_symbols, "%2%3") + symbols = string.gsub(symbols, "%s", "") + end + end + + lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|', + rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score)) + + if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then + + if rule.extended == false then + common.yield_result(task, rule, symbols, spam_score) + common.save_cache(task, digest, rule, symbols, spam_score) + else + local symbols_table = lua_util.str_split(symbols, ",") + lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table) + + common.yield_result(task, rule, symbols_table, spam_score) + common.save_cache(task, digest, rule, symbols_table, spam_score) + end + else + common.save_cache(task, digest, rule, 'OK') + common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + upstream = upstream, + timeout = rule['timeout'], + data = request_data, + callback = spamassassin_callback, + }) + end + + if common.condition_check_and_continue(task, content, rule, digest, spamassassin_check_uncached) then + return + else + spamassassin_check_uncached() + end + +end + +return { + type = { N, 'spam', 'scanner' }, + description = 'spamassassin spam scanner', + configure = spamassassin_config, + check = spamassassin_check, + name = N +} diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua new file mode 100644 index 0000000..826573a --- /dev/null +++ b/lualib/lua_scanners/vadesecure.lua @@ -0,0 +1,351 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module vadesecure +-- This module contains Vadesecure Filterd interface +--]] + +local lua_util = require "lua_util" +local http = require "rspamd_http" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local ucl = require "ucl" +local common = require "lua_scanners/common" + +local N = 'vadesecure' + +local function vade_config(opts) + + local vade_conf = { + name = N, + default_port = 23808, + url = '/api/v1/scan', + use_https = false, + timeout = 5.0, + log_clean = false, + retransmits = 1, + cache_expire = 7200, -- expire redis in 2h + message = '${SCANNER}: spam message found: "${VIRUS}"', + detection_category = "hash", + default_score = 1, + action = false, + log_spamcause = true, + symbol_fail = 'VADE_FAIL', + symbol = 'VADE_CHECK', + settings_outbound = nil, -- Set when there is a settings id for outbound messages + symbols = { + clean = { + symbol = 'VADE_CLEAN', + score = -0.5, + description = 'VadeSecure decided message to be clean' + }, + spam = { + high = { + symbol = 'VADE_SPAM_HIGH', + score = 8.0, + description = 'VadeSecure decided message to be clearly spam' + }, + medium = { + symbol = 'VADE_SPAM_MEDIUM', + score = 5.0, + description = 'VadeSecure decided message to be highly likely spam' + }, + low = { + symbol = 'VADE_SPAM_LOW', + score = 2.0, + description = 'VadeSecure decided message to be likely spam' + }, + }, + malware = { + symbol = 'VADE_MALWARE', + score = 8.0, + description = 'VadeSecure decided message to be malware' + }, + scam = { + symbol = 'VADE_SCAM', + score = 7.0, + description = 'VadeSecure decided message to be scam' + }, + phishing = { + symbol = 'VADE_PHISHING', + score = 8.0, + description = 'VadeSecure decided message to be phishing' + }, + commercial = { + symbol = 'VADE_COMMERCIAL', + score = 0.0, + description = 'VadeSecure decided message to be commercial message' + }, + community = { + symbol = 'VADE_COMMUNITY', + score = 0.0, + description = 'VadeSecure decided message to be community message' + }, + transactional = { + symbol = 'VADE_TRANSACTIONAL', + score = 0.0, + description = 'VadeSecure decided message to be transactional message' + }, + suspect = { + symbol = 'VADE_SUSPECT', + score = 3.0, + description = 'VadeSecure decided message to be suspicious message' + }, + bounce = { + symbol = 'VADE_BOUNCE', + score = 0.0, + description = 'VadeSecure decided message to be bounce message' + }, + other = 'VADE_OTHER', + } + } + + vade_conf = lua_util.override_defaults(vade_conf, opts) + + if not vade_conf.prefix then + vade_conf.prefix = 'rs_' .. vade_conf.name .. '_' + end + + if not vade_conf.log_prefix then + if vade_conf.name:lower() == vade_conf.type:lower() then + vade_conf.log_prefix = vade_conf.name + else + vade_conf.log_prefix = vade_conf.name .. ' (' .. vade_conf.type .. ')' + end + end + + if not vade_conf.servers and vade_conf.socket then + vade_conf.servers = vade_conf.socket + end + + if not vade_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + vade_conf.upstreams = upstream_list.create(rspamd_config, + vade_conf.servers, + vade_conf.default_port) + + if vade_conf.upstreams then + lua_util.add_debug_alias('external_services', vade_conf.name) + return vade_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + vade_conf['servers']) + return nil +end + +local function vade_check(task, content, digest, rule, maybe_part) + local function vade_check_uncached() + local function vade_url(addr) + local url + if rule.use_https then + url = string.format('https://%s:%d%s', tostring(addr), + rule.default_port, rule.url) + else + url = string.format('http://%s:%d%s', tostring(addr), + rule.default_port, rule.url) + end + + return url + end + + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + + local url = vade_url(addr) + local hdrs = {} + + local helo = task:get_helo() + if helo then + hdrs['X-Helo'] = helo + end + local mail_from = task:get_from('smtp') or {} + if mail_from[1] and #mail_from[1].addr > 1 then + hdrs['X-Mailfrom'] = mail_from[1].addr + end + + local rcpt_to = task:get_recipients('smtp') + if rcpt_to then + hdrs['X-Rcptto'] = {} + for _, r in ipairs(rcpt_to) do + table.insert(hdrs['X-Rcptto'], r.addr) + end + end + + local fip = task:get_from_ip() + if fip and fip:is_valid() then + hdrs['X-Inet'] = tostring(fip) + end + + if rule.settings_outbound then + local settings_id = task:get_settings_id() + + if settings_id then + local lua_settings = require "lua_settings" + -- Convert to string + settings_id = lua_settings.settings_by_id(settings_id) + + if settings_id then + settings_id = settings_id.name or '' + + if settings_id == rule.settings_outbound then + lua_util.debugm(rule.name, task, '%s settings has matched outbound', + settings_id) + hdrs['X-Params'] = 'mode=smtpout' + end + end + end + end + + local request_data = { + task = task, + url = url, + body = task:get_content(), + headers = hdrs, + timeout = rule.timeout, + } + + local function vade_callback(http_err, code, body, headers) + + local function vade_requery() + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, + '%s: Request Error: %s - retries left: %s', + rule.log_prefix, http_err, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + url = vade_url(addr) + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + request_data.url = url + + http.request(request_data) + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. + 'retransmits exceed') + end + end + + if http_err then + vade_requery() + else + -- Parse the response + if upstream then + upstream:ok() + end + if code ~= 200 then + rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) + return + end + local parser = ucl.parser() + local ret, err = parser:parse_string(body) + if not ret then + rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body) + task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err) + return + end + local obj = parser:get_object() + local verdict = obj.verdict + if not verdict then + rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj) + task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict') + return + end + local vparts = lua_util.str_split(verdict, ":") + verdict = table.remove(vparts, 1) or verdict + + local sym = rule.symbols[verdict] + if not sym then + sym = rule.symbols.other + end + + if not sym.symbol then + -- Subcategory match + local lvl = 'low' + if vparts and vparts[1] then + lvl = vparts[1] + end + + if sym[lvl] then + sym = sym[lvl] + else + sym = rule.symbols.other + end + end + + local opts = {} + if obj.score then + table.insert(opts, 'score=' .. obj.score) + end + if obj.elapsed then + table.insert(opts, 'elapsed=' .. obj.elapsed) + end + + if rule.log_spamcause and obj.spamcause then + rspamd_logger.infox(task, 'vadesecure verdict="%s", score=%s, spamcause="%s", message-id="%s"', + verdict, obj.score, obj.spamcause, task:get_message_id()) + else + lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"', + verdict, obj.score, obj.spamcause) + end + + if #vparts > 0 then + table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':')) + end + + task:insert_result(sym.symbol, 1.0, opts) + end + end + + request_data.callback = vade_callback + http.request(request_data) + end + + if common.condition_check_and_continue(task, content, rule, digest, + vade_check_uncached, maybe_part) then + return + else + vade_check_uncached() + end + +end + +return { + type = { 'vadesecure', 'scanner' }, + description = 'VadeSecure Filterd interface', + configure = vade_config, + check = vade_check, + name = N +} diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua new file mode 100644 index 0000000..d937c41 --- /dev/null +++ b/lualib/lua_scanners/virustotal.lua @@ -0,0 +1,214 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module virustotal +-- This module contains Virustotal integration support +-- https://www.virustotal.com/ +--]] + +local lua_util = require "lua_util" +local http = require "rspamd_http" +local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" + +local N = 'virustotal' + +local function virustotal_config(opts) + + local default_conf = { + name = N, + url = 'https://www.virustotal.com/vtapi/v2/file', + timeout = 5.0, + log_clean = false, + retransmits = 1, + cache_expire = 7200, -- expire redis in 2h + message = '${SCANNER}: spam message found: "${VIRUS}"', + detection_category = "virus", + default_score = 1, + action = false, + scan_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + apikey = nil, -- Required to set by user + -- Specific for virustotal + minimum_engines = 3, -- Minimum required to get scored + full_score_engines = 7, -- After this number we set max score + } + + default_conf = lua_util.override_defaults(default_conf, opts) + + if not default_conf.prefix then + default_conf.prefix = 'rs_' .. default_conf.name .. '_' + end + + if not default_conf.log_prefix then + if default_conf.name:lower() == default_conf.type:lower() then + default_conf.log_prefix = default_conf.name + else + default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')' + end + end + + if not default_conf.apikey then + rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks') + + return nil + end + + lua_util.add_debug_alias('external_services', default_conf.name) + return default_conf +end + +local function virustotal_check(task, content, digest, rule, maybe_part) + local function virustotal_check_uncached() + local function make_url(hash) + return string.format('%s/report?apikey=%s&resource=%s', + rule.url, rule.apikey, hash) + end + + local hash = rspamd_cryptobox_hash.create_specific('md5') + hash:update(content) + hash = hash:hex() + + local url = make_url(hash) + lua_util.debugm(N, task, "send request %s", url) + local request_data = { + task = task, + url = url, + timeout = rule.timeout, + } + + local function vt_http_callback(http_err, code, body, headers) + if http_err then + rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers) + else + local cached + local dyn_score + -- Parse the response + if code ~= 200 then + if code == 404 then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: hash %s clean (not found)', + rule.log_prefix, hash) + else + lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)', + rule.log_prefix, hash) + end + elseif code == 204 then + -- Request rate limit exceeded + rspamd_logger.infox(task, 'virustotal request rate limit exceeded') + task:insert_result(rule.symbol_fail, 1.0, 'rate limit exceeded') + return + else + rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) + return + end + else + local ucl = require "ucl" + local parser = ucl.parser() + local res, json_err = parser:parse_string(body) + + lua_util.debugm(rule.name, task, '%s: got reply data: "%s"', + rule.log_prefix, body) + + if res then + local obj = parser:get_object() + if not obj.positives or type(obj.positives) ~= 'number' then + if obj.response_code then + if obj.response_code == 0 then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: hash %s clean (not found)', + rule.log_prefix, hash) + else + lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)', + rule.log_prefix, hash) + end + else + rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', + 'bad response code: ' .. tostring(obj.response_code), body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element') + return + end + else + rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', + 'no response_code', body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element') + return + end + else + if obj.positives < rule.minimum_engines then + lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min', + rule.log_prefix, obj.positives, rule.minimum_engines) + -- TODO: add proper hashing! + cached = 'OK' + else + if obj.positives > rule.full_score_engines then + dyn_score = 1.0 + else + local norm_pos = obj.positives - rule.minimum_engines + dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines) + end + + if dyn_score < 0 or dyn_score > 1 then + dyn_score = 1.0 + end + local sopt = string.format("%s:%s/%s", + hash, obj.positives, obj.total) + common.yield_result(task, rule, sopt, dyn_score, nil, maybe_part) + cached = sopt + end + end + else + -- not res + rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', + json_err, body, headers) + task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err) + return + end + end + + if cached then + common.save_cache(task, digest, rule, cached, dyn_score, maybe_part) + end + end + end + + request_data.callback = vt_http_callback + http.request(request_data) + end + + if common.condition_check_and_continue(task, content, rule, digest, + virustotal_check_uncached) then + return + else + + virustotal_check_uncached() + end + +end + +return { + type = 'antivirus', + description = 'Virustotal integration', + configure = virustotal_config, + check = virustotal_check, + name = N +} |