summaryrefslogtreecommitdiffstats
path: root/lualib/lua_scanners
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
commit133a45c109da5310add55824db21af5239951f93 (patch)
treeba6ac4c0a950a0dda56451944315d66409923918 /lualib/lua_scanners
parentInitial commit. (diff)
downloadrspamd-upstream.tar.xz
rspamd-upstream.zip
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--lualib/lua_scanners/avast.lua304
-rw-r--r--lualib/lua_scanners/clamav.lua193
-rw-r--r--lualib/lua_scanners/cloudmark.lua372
-rw-r--r--lualib/lua_scanners/common.lua539
-rw-r--r--lualib/lua_scanners/dcc.lua313
-rw-r--r--lualib/lua_scanners/fprot.lua181
-rw-r--r--lualib/lua_scanners/icap.lua713
-rw-r--r--lualib/lua_scanners/init.lua75
-rw-r--r--lualib/lua_scanners/kaspersky_av.lua197
-rw-r--r--lualib/lua_scanners/kaspersky_se.lua287
-rw-r--r--lualib/lua_scanners/oletools.lua369
-rw-r--r--lualib/lua_scanners/p0f.lua227
-rw-r--r--lualib/lua_scanners/pyzor.lua206
-rw-r--r--lualib/lua_scanners/razor.lua181
-rw-r--r--lualib/lua_scanners/savapi.lua261
-rw-r--r--lualib/lua_scanners/sophos.lua192
-rw-r--r--lualib/lua_scanners/spamassassin.lua213
-rw-r--r--lualib/lua_scanners/vadesecure.lua351
-rw-r--r--lualib/lua_scanners/virustotal.lua214
19 files changed, 5388 insertions, 0 deletions
diff --git a/lualib/lua_scanners/avast.lua b/lualib/lua_scanners/avast.lua
new file mode 100644
index 0000000..7e77897
--- /dev/null
+++ b/lualib/lua_scanners/avast.lua
@@ -0,0 +1,304 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module avast
+-- This module contains avast av access functions
+--]]
+
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_regexp = require "rspamd_regexp"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "avast"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function avast_config(opts)
+ local avast_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ timeout = 4.0, -- FIXME: this will break task_timeout!
+ log_clean = false,
+ detection_category = "virus",
+ retransmits = 1,
+ servers = nil, -- e.g. /var/run/avast/scan.sock
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ tmpdir = '/tmp',
+ }
+
+ avast_conf = lua_util.override_defaults(avast_conf, opts)
+
+ if not avast_conf.prefix then
+ avast_conf.prefix = 'rs_' .. avast_conf.name .. '_'
+ end
+
+ if not avast_conf.log_prefix then
+ if avast_conf.name:lower() == avast_conf.type:lower() then
+ avast_conf.log_prefix = avast_conf.name
+ else
+ avast_conf.log_prefix = avast_conf.name .. ' (' .. avast_conf.type .. ')'
+ end
+ end
+
+ if not avast_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers/unix socket defined')
+
+ return nil
+ end
+
+ avast_conf['upstreams'] = upstream_list.create(rspamd_config,
+ avast_conf['servers'],
+ 0)
+
+ if avast_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', avast_conf.name)
+ return avast_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ avast_conf['servers'])
+ return nil
+end
+
+local function avast_check(task, content, digest, rule, maybe_part)
+ local function avast_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local CRLF = '\r\n'
+
+ -- Common tcp options
+ local tcp_opts = {
+ stop_pattern = CRLF,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ task = task
+ }
+
+ -- Regexps to process reply from avast
+ local clean_re = rspamd_regexp.create_cached(
+ [=[(?!\\)\t\[\+\]]=]
+ )
+ local virus_re = rspamd_regexp.create_cached(
+ [[(?!\\)\t\[L\]\d\.\d\t\d\s(.*)]]
+ )
+ local error_re = rspamd_regexp.create_cached(
+ [[(?!\\)\t\[E\]\d+\.0\tError\s\d+\s(.*)]]
+ )
+
+ -- Used to make a dialog
+ local tcp_conn
+
+ -- Save content in file as avast can work with files only
+ local fname = string.format('%s/%s.avtmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for avast scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure file cleanup on task processed
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ -- Dialog stages closures
+ local avast_helo_cb
+ local avast_scan_cb
+ local avast_scan_done_cb
+
+ -- Utility closures
+ local function maybe_retransmit()
+ if retransmits > 0 then
+ retransmits = retransmits - 1
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+
+ return
+ end
+
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ tcp_opts.upstream = upstream
+ tcp_opts.callback = avast_helo_cb
+
+ local is_succ, err = tcp.request(tcp_opts)
+
+ if not is_succ then
+ rspamd_logger.infox(task, 'cannot create connection to avast server: %s (%s)',
+ addr:to_string(true), err)
+ else
+ lua_util.debugm(rule.log_prefix, task, 'established connection to %s; retransmits=%s',
+ addr:to_string(true), retransmits)
+ end
+ end
+
+ local function no_connection_error(err)
+ if err then
+ if tcp_conn then
+ tcp_conn:close()
+ tcp_conn = nil
+
+ rspamd_logger.infox(task, 'failed to request to avast (%s): %s',
+ addr:to_string(true), err)
+ maybe_retransmit()
+ end
+
+ return false
+ end
+
+ return true
+ end
+
+
+ -- Define callbacks
+ avast_helo_cb = function(merr, mdata, conn)
+ -- Called when we have established a connection but not read anything
+ tcp_conn = conn
+
+ if no_connection_error(merr) then
+ -- Check mdata to ensure that it starts with 220
+ if #mdata > 3 and tostring(mdata:span(1, 3)) == '220' then
+ tcp_conn:add_write(avast_scan_cb, string.format(
+ 'SCAN %s%s', fname, CRLF))
+ else
+ rspamd_logger.errx(task, 'Unhandled response: %s', mdata)
+ end
+ end
+ end
+
+ avast_scan_cb = function(merr)
+ -- Called when we have send request to avast and are waiting for reply
+ if no_connection_error(merr) then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ end
+
+ avast_scan_done_cb = function(merr, mdata)
+ if no_connection_error(merr) then
+ lua_util.debugm(rule.log_prefix, task, 'got reply from avast: %s',
+ mdata)
+ if #mdata > 4 then
+ local beg = tostring(mdata:span(1, 3))
+
+ if beg == '210' then
+ -- Ignore 210, fire another read
+ if tcp_conn then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ elseif beg == '200' then
+ -- Final line
+ if tcp_conn then
+ tcp_conn:close()
+ tcp_conn = nil
+ end
+ else
+ -- Check line using regular expressions
+ local cached
+ local ret = clean_re:search(mdata, false, true)
+
+ if ret then
+ cached = 'OK'
+ if rule.log_clean then
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule.symbol, rule.type)
+ end
+ end
+
+ if not cached then
+ ret = virus_re:search(mdata, false, true)
+
+ if ret then
+ local vname = ret[1][2]
+
+ if vname then
+ vname = vname:gsub('\\ ', ' '):gsub('\\\\', '\\')
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ end
+ end
+ end
+
+ if not cached then
+ ret = error_re:search(mdata, false, true)
+
+ if ret then
+ rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, ret[1][2])
+ common.yield_result(task, rule, 'error:' .. ret[1][2],
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ else
+ -- Unexpected reply
+ rspamd_logger.errx(task, '%s: unexpected reply: %s', rule.log_prefix, mdata)
+ end
+ -- Read more
+ if tcp_conn then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ end
+ end
+ end
+ end
+
+ -- Send the real request
+ maybe_retransmit()
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ avast_check_uncached, maybe_part) then
+ return
+ else
+ avast_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Avast antivirus',
+ configure = avast_config,
+ check = avast_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
new file mode 100644
index 0000000..fc99ab0
--- /dev/null
+++ b/lualib/lua_scanners/clamav.lua
@@ -0,0 +1,193 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module clamav
+-- This module contains clamav access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "clamav"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function clamav_config(opts)
+ local clamav_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 3310,
+ log_clean = false,
+ timeout = 5.0, -- FIXME: this will break task_timeout!
+ detection_category = "virus",
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ clamav_conf = lua_util.override_defaults(clamav_conf, opts)
+
+ if not clamav_conf.prefix then
+ clamav_conf.prefix = 'rs_' .. clamav_conf.name .. '_'
+ end
+
+ if not clamav_conf.log_prefix then
+ if clamav_conf.name:lower() == clamav_conf.type:lower() then
+ clamav_conf.log_prefix = clamav_conf.name
+ else
+ clamav_conf.log_prefix = clamav_conf.name .. ' (' .. clamav_conf.type .. ')'
+ end
+ end
+
+ if not clamav_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ clamav_conf['upstreams'] = upstream_list.create(rspamd_config,
+ clamav_conf['servers'],
+ clamav_conf.default_port)
+
+ if clamav_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', clamav_conf.name)
+ return clamav_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ clamav_conf['servers'])
+ return nil
+end
+
+local function clamav_check(task, content, digest, rule, maybe_part)
+ local function clamav_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0",
+ #content)
+ local footer = rspamd_util.pack(">I4", 0)
+
+ local function clamav_callback(err, data)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix)
+ common.yield_result(task, rule,
+ 'failed to scan and retransmits exceed', 0.0, 'fail',
+ maybe_part)
+ end
+
+ else
+ data = tostring(data)
+ local cached
+ lua_util.debugm(rule.name, task, '%s: got reply: %s',
+ rule.log_prefix, data)
+ if data == 'stream: OK' then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', rule.log_prefix)
+ end
+ else
+ local vname = string.match(data, 'stream: (.+) FOUND')
+ if string.find(vname, '^Heuristics%.Encrypted') then
+ rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+ common.yield_result(task, rule, 'File is encrypted: ' .. vname,
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then
+ rspamd_logger.errx(task, '%s: ClamAV Found an OLE2 Office Macro', rule.log_prefix)
+ common.yield_result(task, rule, vname, 0.0, 'macro', maybe_part)
+ cached = 'MACRO'
+ elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then
+ rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix)
+ common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0,
+ 'fail', maybe_part)
+ elseif vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ else
+ rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. vname, 0.0,
+ 'fail', maybe_part)
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ upstream = upstream,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ clamav_check_uncached, maybe_part) then
+ return
+ else
+ clamav_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'clamav antivirus',
+ configure = clamav_config,
+ check = clamav_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/cloudmark.lua b/lualib/lua_scanners/cloudmark.lua
new file mode 100644
index 0000000..b07f238
--- /dev/null
+++ b/lualib/lua_scanners/cloudmark.lua
@@ -0,0 +1,372 @@
+--[[
+Copyright (c) 2021, Alexander Moisseev <moiseev@mezonplus.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module cloudmark
+-- This module contains Cloudmark v2 interface
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local rspamd_util = require "rspamd_util"
+local common = require "lua_scanners/common"
+local fun = require "fun"
+local lua_mime = require "lua_mime"
+
+local N = 'cloudmark'
+-- Boundary for multipart transfers, generated on module init
+local static_boundary = rspamd_util.random_hex(32)
+
+local function cloudmark_url(rule, addr, maybe_url)
+ local url
+ local port = addr:get_port()
+
+ maybe_url = maybe_url or rule.url
+ if port == 0 then
+ port = rule.default_port
+ end
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ port, maybe_url)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ port, maybe_url)
+ end
+
+ return url
+end
+
+-- Detect cloudmark max size
+local function cloudmark_preload(rule, cfg, ev_base, _)
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local function max_message_size_cb(http_err, code, body, _)
+ if http_err then
+ rspamd_logger.errx(ev_base, 'HTTP error when getting max message size: %s',
+ http_err)
+ return
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(ev_base, 'bad HTTP code when getting max message size: %s', code)
+ end
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(ev_base, 'could not parse response body [%s]: %s', body, err)
+ return
+ end
+ local obj = parser:get_object()
+ local ms = obj.maxMessageSize
+ if not ms then
+ rspamd_logger.errx(ev_base, 'missing maxMessageSize in the response body (JSON): %s', obj)
+ return
+ end
+
+ rule.max_size = ms
+ lua_util.debugm(N, cfg, 'set maximum message size set to %s bytes', ms)
+ end
+ http.request({
+ ev_base = ev_base,
+ config = cfg,
+ url = cloudmark_url(rule, addr, '/score/v2/max-message-size'),
+ callback = max_message_size_cb,
+ })
+end
+
+local function cloudmark_config(opts)
+
+ local cloudmark_conf = {
+ name = N,
+ default_port = 2713,
+ url = '/score/v2/message',
+ use_https = false,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ score_threshold = 90, -- minimum score to considerate reply
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ max_message = 0,
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ log_spamcause = true,
+ symbol_fail = 'CLOUDMARK_FAIL',
+ symbol = 'CLOUDMARK_CHECK',
+ symbol_spam = 'CLOUDMARK_SPAM',
+ add_headers = false, -- allow addition of the headers from Cloudmark
+ }
+
+ cloudmark_conf = lua_util.override_defaults(cloudmark_conf, opts)
+
+ if not cloudmark_conf.prefix then
+ cloudmark_conf.prefix = 'rs_' .. cloudmark_conf.name .. '_'
+ end
+
+ if not cloudmark_conf.log_prefix then
+ if cloudmark_conf.name:lower() == cloudmark_conf.type:lower() then
+ cloudmark_conf.log_prefix = cloudmark_conf.name
+ else
+ cloudmark_conf.log_prefix = cloudmark_conf.name .. ' (' .. cloudmark_conf.type .. ')'
+ end
+ end
+
+ if not cloudmark_conf.servers and cloudmark_conf.socket then
+ cloudmark_conf.servers = cloudmark_conf.socket
+ end
+
+ if not cloudmark_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ cloudmark_conf.upstreams = upstream_list.create(rspamd_config,
+ cloudmark_conf.servers,
+ cloudmark_conf.default_port)
+
+ if cloudmark_conf.upstreams then
+
+ cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } }
+ cloudmark_conf.preloads = { cloudmark_preload }
+ lua_util.add_debug_alias('external_services', cloudmark_conf.name)
+ return cloudmark_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ cloudmark_conf['servers'])
+ return nil
+end
+
+-- Converts a key-value map to the table representing multipart body, with the following values:
+-- `data`: data of the part
+-- `filename`: optional filename
+-- `content-type`: content type of the element (optional)
+-- `content-transfer-encoding`: optional CTE header
+local function table_to_multipart_body(tbl, boundary)
+ local seen_data = false
+ local out = {}
+
+ for k, v in pairs(tbl) do
+ if v.data then
+ seen_data = true
+ table.insert(out, string.format('--%s\r\n', boundary))
+ if v.filename then
+ table.insert(out,
+ string.format('Content-Disposition: form-data; name="%s"; filename="%s"\r\n',
+ k, v.filename))
+ else
+ table.insert(out,
+ string.format('Content-Disposition: form-data; name="%s"\r\n', k))
+ end
+ if v['content-type'] then
+ table.insert(out,
+ string.format('Content-Type: %s\r\n', v['content-type']))
+ else
+ table.insert(out, 'Content-Type: text/plain\r\n')
+ end
+ if v['content-transfer-encoding'] then
+ table.insert(out,
+ string.format('Content-Transfer-Encoding: %s\r\n',
+ v['content-transfer-encoding']))
+ else
+ table.insert(out, 'Content-Transfer-Encoding: binary\r\n')
+ end
+ table.insert(out, '\r\n')
+ table.insert(out, v.data)
+ table.insert(out, '\r\n')
+ end
+ end
+
+ if seen_data then
+ table.insert(out, string.format('--%s--\r\n', boundary))
+ end
+
+ return out
+end
+
+local function parse_cloudmark_reply(task, rule, body)
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
+ return
+ end
+ local obj = parser:get_object()
+ lua_util.debugm(N, task, 'cloudmark reply is: %s', obj)
+
+ if not obj.score then
+ rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: no score')
+ return
+ end
+
+ if obj.analysis then
+ -- Report analysis string
+ rspamd_logger.infox(task, 'cloudmark report string: %s', obj.analysis)
+ end
+
+ local score = tonumber(obj.score) or 0
+ if score >= rule.score_threshold then
+ task:insert_result(rule.symbol_spam, 1.0, tostring(score))
+ end
+
+ if rule.add_headers and type(obj.appendHeaders) == 'table' then
+ local headers_add = fun.tomap(fun.map(function(h)
+ return h.headerField, {
+ order = 1, value = h.body
+ }
+ end, obj.appendHeaders))
+ lua_mime.modify_headers(task, {
+ add = headers_add
+ })
+ end
+
+end
+
+local function cloudmark_check(task, content, digest, rule, maybe_part)
+ local function cloudmark_check_uncached()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = cloudmark_url(rule, addr)
+ local message_data = task:get_content()
+ if rule.max_message and rule.max_message > 0 and #message_data > rule.max_message then
+ task:insert_result(rule['symbol_fail'], 0.0, 'Message too large: ' .. #message_data)
+ return
+ end
+ local request = {
+ rfc822 = {
+ ['Content-Type'] = 'message/rfc822',
+ data = message_data,
+ }
+ }
+
+ local helo = task:get_helo()
+ if helo then
+ request['heloDomain'] = {
+ data = helo,
+ }
+ end
+ local mail_from = task:get_from('smtp') or {}
+ if mail_from[1] and #mail_from[1].addr > 1 then
+ request['mailFrom'] = {
+ data = mail_from[1].addr
+ }
+ end
+
+ local rcpt_to = task:get_recipients('smtp')
+ if rcpt_to then
+ request['rcptTo'] = {
+ data = table.concat(fun.totable(fun.map(function(r)
+ return r.addr
+ end, rcpt_to)), ',')
+ }
+ end
+
+ local fip = task:get_from_ip()
+ if fip and fip:is_valid() then
+ request['connIp'] = tostring(fip)
+ end
+
+ local hostname = task:get_hostname()
+ if hostname then
+ request['fromHost'] = hostname
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = table_to_multipart_body(request, static_boundary),
+ headers = {
+ ['Content-Type'] = string.format('multipart/form-data; boundary="%s"', static_boundary)
+ },
+ timeout = rule.timeout,
+ }
+
+ local function cloudmark_callback(http_err, code, body, headers)
+
+ local function cloudmark_requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = cloudmark_url(rule, addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ upstream:fail()
+ end
+ end
+
+ if http_err then
+ cloudmark_requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ parse_cloudmark_reply(task, rule, body)
+ end
+ end
+
+ request_data.callback = cloudmark_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ cloudmark_check_uncached, maybe_part) then
+ return
+ else
+ cloudmark_check_uncached()
+ end
+end
+
+return {
+ type = { 'cloudmark', 'scanner' },
+ description = 'Cloudmark cartridge interface',
+ configure = cloudmark_config,
+ check = cloudmark_check,
+ name = N,
+}
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
new file mode 100644
index 0000000..11f5e1f
--- /dev/null
+++ b/lualib/lua_scanners/common.lua
@@ -0,0 +1,539 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_scanners_common
+-- This module contains common external scanners functions
+--]]
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local lua_magic_types = require "lua_magic/types"
+local fun = require "fun"
+
+local exports = {}
+
+local function log_clean(task, rule, msg)
+
+ msg = msg or 'message or mime_part is clean'
+
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
+ else
+ lua_util.debugm(rule.name, task, '%s: %s', rule.log_prefix, msg)
+ end
+
+end
+
+local function match_patterns(default_sym, found, patterns, dyn_weight)
+ if type(patterns) ~= 'table' then
+ return default_sym, dyn_weight
+ end
+ if not patterns[1] then
+ for sym, pat in pairs(patterns) do
+ if pat:match(found) then
+ return sym, '1'
+ end
+ end
+ return default_sym, dyn_weight
+ else
+ for _, p in ipairs(patterns) do
+ for sym, pat in pairs(p) do
+ if pat:match(found) then
+ return sym, '1'
+ end
+ end
+ end
+ return default_sym, dyn_weight
+ end
+end
+
+local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part)
+ local all_whitelisted = true
+ local patterns
+ local symbol
+ local threat_table
+ local threat_info
+ local flags
+
+ if type(vname) == 'string' then
+ threat_table = { vname }
+ elseif type(vname) == 'table' then
+ threat_table = vname
+ end
+
+
+ -- This should be more generic
+ if not is_fail then
+ patterns = rule.patterns
+ symbol = rule.symbol
+ threat_info = rule.detection_category .. 'found'
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
+ elseif is_fail == 'fail' then
+ patterns = rule.patterns_fail
+ symbol = rule.symbol_fail
+ threat_info = "FAILED with error"
+ dyn_weight = 0.0
+ elseif is_fail == 'encrypted' then
+ patterns = rule.patterns
+ symbol = rule.symbol_encrypted
+ threat_info = "Scan has returned that input was encrypted"
+ dyn_weight = 1.0
+ elseif is_fail == 'macro' then
+ patterns = rule.patterns
+ symbol = rule.symbol_macro
+ threat_info = "Scan has returned that input contains macros"
+ dyn_weight = 1.0
+ end
+
+ for _, tm in ipairs(threat_table) do
+ local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight)
+ if rule.whitelist and rule.whitelist:get_key(tm) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, tm)
+ else
+ all_whitelisted = false
+ rspamd_logger.infox(task, '%s: result - %s: "%s - score: %s"',
+ rule.log_prefix, threat_info, tm, symscore)
+
+ if maybe_part and rule.show_attachments and maybe_part:get_filename() then
+ local fname = maybe_part:get_filename()
+ task:insert_result(symname, symscore, string.format("%s|%s",
+ tm, fname))
+ else
+ task:insert_result(symname, symscore, tm)
+ end
+
+ end
+ end
+
+ if rule.action and is_fail ~= 'fail' and not all_whitelisted then
+ threat_table = table.concat(threat_table, '; ')
+ if rule.action ~= 'reject' then
+ flags = 'least'
+ end
+ task:set_pre_result(rule.action,
+ lua_util.template(rule.message or 'Rejected', {
+ SCANNER = rule.name,
+ VIRUS = threat_table,
+ }), rule.name, nil, nil, flags)
+ end
+end
+
+local function message_not_too_large(task, content, rule)
+ local max_size = tonumber(rule.max_size)
+ if not max_size then
+ return true
+ end
+ if #content > max_size then
+ rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
+ rule.log_prefix, #content, max_size)
+ return false
+ end
+ return true
+end
+
+local function message_not_too_small(task, content, rule)
+ local min_size = tonumber(rule.min_size)
+ if not min_size then
+ return true
+ end
+ if #content < min_size then
+ rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
+ rule.log_prefix, #content, min_size)
+ return false
+ end
+ return true
+end
+
+local function message_min_words(task, rule)
+ if rule.text_part_min_words and tonumber(rule.text_part_min_words) > 0 then
+ local text_part_above_limit = false
+ local text_parts = task:get_text_parts()
+
+ local filter_func = function(p)
+ return p:get_words_count() >= tonumber(rule.text_part_min_words)
+ end
+
+ fun.each(function(p)
+ text_part_above_limit = true
+ end, fun.filter(filter_func, text_parts))
+
+ if not text_part_above_limit then
+ rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s',
+ rule.log_prefix, rule.text_part_min_words)
+ end
+
+ return text_part_above_limit
+ else
+ return true
+ end
+end
+
+local function dynamic_scan(task, rule)
+ if rule.dynamic_scan then
+ if rule.action ~= 'reject' then
+ local metric_result = task:get_metric_score()
+ local metric_action = task:get_metric_action()
+ local has_pre_result = task:has_pre_result()
+ -- ToDo: needed?
+ -- Sometimes leads to FPs
+ --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+ -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
+ -- return false
+ --elseif metric_result[1] > metric_result[2]*2 then
+ if metric_result[1] > metric_result[2] * 2 then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
+ return false
+ elseif has_pre_result and metric_action == 'reject' then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set')
+ return false
+ else
+ return true, 'undecided'
+ end
+ else
+ return true, 'dynamic_scan is not possible with config `action=reject;`'
+ end
+ else
+ return true
+ end
+end
+
+local function need_check(task, content, rule, digest, fn, maybe_part)
+
+ local uncached = true
+ local key = digest
+
+ local function redis_av_cb(err, data)
+ if data and type(data) == 'string' then
+ -- Cached
+ data = lua_util.str_split(data, '\t')
+ local threat_string = lua_util.str_split(data[1], '\v')
+ local score = data[2] or rule.default_score
+
+ if threat_string[1] ~= 'OK' then
+ if threat_string[1] == 'MACRO' then
+ yield_result(task, rule, 'File contains macros',
+ 0.0, 'macro', maybe_part)
+ elseif threat_string[1] == 'ENCRYPTED' then
+ yield_result(task, rule, 'File is encrypted',
+ 0.0, 'encrypted', maybe_part)
+ else
+ lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s',
+ rule.log_prefix, key, threat_string[1], score)
+ yield_result(task, rule, threat_string, score, false, maybe_part)
+ end
+
+ else
+ lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
+ rule.log_prefix, key, threat_string[1])
+ end
+ uncached = false
+ else
+ if err then
+ rspamd_logger.errx(task, 'got error checking cache: %s', err)
+ end
+ end
+
+ local f_message_not_too_large = message_not_too_large(task, content, rule)
+ local f_message_not_too_small = message_not_too_small(task, content, rule)
+ local f_message_min_words = message_min_words(task, rule)
+ local f_dynamic_scan = dynamic_scan(task, rule)
+
+ if uncached and
+ f_message_not_too_large and
+ f_message_not_too_small and
+ f_message_min_words and
+ f_dynamic_scan then
+
+ fn()
+
+ end
+
+ end
+
+ if rule.redis_params and not rule.no_cache then
+
+ key = rule.prefix .. key
+
+ if lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_av_cb, --callback
+ 'GET', -- command
+ { key } -- arguments)
+ ) then
+ return true
+ end
+ end
+
+ return false
+
+end
+
+local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part)
+ local key = digest
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
+
+ local function redis_set_cb(err)
+ -- Do nothing
+ if err then
+ rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
+ rule.detection_category, to_save, key, err)
+ else
+ lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s',
+ rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
+ end
+ end
+
+ if type(to_save) == 'table' then
+ to_save = table.concat(to_save, '\v')
+ end
+
+ local value_tbl = { to_save, dyn_weight }
+ if maybe_part and rule.show_attachments and maybe_part:get_filename() then
+ local fname = maybe_part:get_filename()
+ table.insert(value_tbl, fname)
+ end
+ local value = table.concat(value_tbl, '\t')
+
+ if rule.redis_params and rule.prefix then
+ key = rule.prefix .. key
+
+ lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, rule.cache_expire or 0, value }
+ )
+ end
+
+ return false
+end
+
+local function create_regex_table(patterns)
+ local regex_table = {}
+ if patterns[1] then
+ for i, p in ipairs(patterns) do
+ if type(p) == 'table' then
+ local new_set = {}
+ for k, v in pairs(p) do
+ new_set[k] = rspamd_regexp.create_cached(v)
+ end
+ regex_table[i] = new_set
+ else
+ regex_table[i] = {}
+ end
+ end
+ else
+ for k, v in pairs(patterns) do
+ regex_table[k] = rspamd_regexp.create_cached(v)
+ end
+ end
+ return regex_table
+end
+
+local function match_filter(task, rule, found, patterns, pat_type)
+ if type(patterns) ~= 'table' or not found then
+ return false
+ end
+ if not patterns[1] then
+ for _, pat in pairs(patterns) do
+ if pat_type == 'ext' and tostring(pat) == tostring(found) then
+ return true
+ elseif pat_type == 'regex' and pat:match(found) then
+ return true
+ end
+ end
+ return false
+ else
+ for _, p in ipairs(patterns) do
+ for _, pat in ipairs(p) do
+ if pat_type == 'ext' and tostring(pat) == tostring(found) then
+ return true
+ elseif pat_type == 'regex' and pat:match(found) then
+ return true
+ end
+ end
+ end
+ return false
+ end
+end
+
+-- borrowed from mime_types.lua
+-- ext is the last extension, LOWERCASED
+-- ext2 is the one before last extension LOWERCASED
+local function gen_extension(fname)
+ local filename_parts = lua_util.str_split(fname, '.')
+
+ local ext = {}
+ for n = 1, 2 do
+ ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+ end
+ return ext[1], ext[2], filename_parts
+end
+
+local function check_parts_match(task, rule)
+
+ local filter_func = function(p)
+ local mtype, msubtype = p:get_type()
+ local detected_ext = p:get_detected_ext()
+ local fname = p:get_filename()
+ local ext, ext2
+
+ if rule.scan_all_mime_parts == false then
+ -- check file extension and filename regex matching
+ --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname)
+ if fname ~= nil then
+ ext, ext2 = gen_extension(fname)
+ --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname)
+ if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
+ or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2)
+ return true
+ elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix)
+ return true
+ end
+ end
+ -- check content type string regex matching
+ if mtype ~= nil and msubtype ~= nil then
+ local ct = string.format('%s/%s', mtype, msubtype):lower()
+ if match_filter(task, rule, ct, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: regex content-type: %s', rule.log_prefix, ct)
+ return true
+ end
+ end
+ -- check detected content type (libmagic) regex matching
+ if detected_ext then
+ local magic = lua_magic_types[detected_ext] or {}
+ if match_filter(task, rule, detected_ext, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: detected extension matched: |%s|', rule.log_prefix, detected_ext)
+ return true
+ elseif magic.ct and match_filter(task, rule, magic.ct, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: regex detected libmagic content-type: %s',
+ rule.log_prefix, magic.ct)
+ return true
+ end
+ end
+ -- check filenames in archives
+ if p:is_archive() then
+ local arch = p:get_archive()
+ local filelist = arch:get_files_full(1000)
+ for _, f in ipairs(filelist) do
+ ext, ext2 = gen_extension(f.name)
+ if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
+ or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2)
+ --lua_util.debugm(rule.name, task, '%s: extension matched in archive: %s', rule.log_prefix, ext)
+ return true
+ elseif match_filter(task, rule, f.name, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: filename regex matched in archive', rule.log_prefix)
+ return true
+ end
+ end
+ end
+ end
+
+ -- check text_part has more words than text_part_min_words_check
+ if rule.scan_text_mime and rule.text_part_min_words and p:is_text() and
+ p:get_words_count() >= tonumber(rule.text_part_min_words) then
+ return true
+ end
+
+ if rule.scan_image_mime and p:is_image() then
+ return true
+ end
+
+ if rule.scan_all_mime_parts ~= false then
+ local is_part_checkable = (p:is_attachment() and (not p:is_image() or rule.scan_image_mime))
+ if detected_ext then
+ -- We know what to scan!
+ local magic = lua_magic_types[detected_ext] or {}
+
+ if magic.av_check ~= false or is_part_checkable then
+ return true
+ end
+ elseif is_part_checkable then
+ -- Just rely on attachment property
+ return true
+ end
+ end
+
+ return false
+ end
+
+ return fun.filter(filter_func, task:get_parts())
+end
+
+local function check_metric_results(task, rule)
+
+ if rule.action ~= 'reject' then
+ local metric_result = task:get_metric_score()
+ local metric_action = task:get_metric_action()
+ local has_pre_result = task:has_pre_result()
+
+ if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+ return true, 'result is already reject'
+ elseif metric_result[1] > metric_result[2] * 2 then
+ return true, 'score > 2 * reject_level: ' .. metric_result[1]
+ elseif has_pre_result and metric_action == 'reject' then
+ return true, 'pre_result reject is set'
+ else
+ return false, 'undecided'
+ end
+ else
+ return false, 'dynamic_scan is not possible with config `action=reject;`'
+ end
+end
+
+exports.log_clean = log_clean
+exports.yield_result = yield_result
+exports.match_patterns = match_patterns
+exports.condition_check_and_continue = need_check
+exports.save_cache = save_cache
+exports.create_regex_table = create_regex_table
+exports.check_parts_match = check_parts_match
+exports.check_metric_results = check_metric_results
+
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ print('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ print('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
new file mode 100644
index 0000000..8d5e9e1
--- /dev/null
+++ b/lualib/lua_scanners/dcc.lua
@@ -0,0 +1,313 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module dcc
+-- This module contains dcc access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+local fun = require "fun"
+
+local N = 'dcc'
+
+local function dcc_config(opts)
+
+ local dcc_conf = {
+ name = N,
+ default_port = 10045,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: bulk message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ client = '0.0.0.0',
+ symbol_fail = 'DCC_FAIL',
+ symbol = 'DCC_REJECT',
+ symbol_bulk = 'DCC_BULK',
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ }
+
+ dcc_conf = lua_util.override_defaults(dcc_conf, opts)
+
+ if not dcc_conf.prefix then
+ dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_'
+ end
+
+ if not dcc_conf.log_prefix then
+ dcc_conf.log_prefix = dcc_conf.name
+ end
+
+ if not dcc_conf.servers and dcc_conf.socket then
+ dcc_conf.servers = dcc_conf.socket
+ end
+
+ if not dcc_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ dcc_conf.upstreams = upstream_list.create(rspamd_config,
+ dcc_conf.servers,
+ dcc_conf.default_port)
+
+ if dcc_conf.upstreams then
+ lua_util.add_debug_alias('external_services', dcc_conf.name)
+ return dcc_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ dcc_conf['servers'])
+ return nil
+end
+
+local function dcc_check(task, content, digest, rule)
+ local function dcc_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local client = rule.client
+
+ local client_ip = task:get_from_ip()
+ if client_ip and client_ip:is_valid() then
+ client = client_ip:to_string()
+ end
+ local client_host = task:get_hostname()
+ if client_host then
+ client = client .. "\r" .. client_host
+ end
+
+ -- HELO
+ local helo = task:get_helo() or ''
+
+ -- Envelope From
+ local ef = task:get_from()
+ local envfrom = 'test@example.com'
+ if ef and ef[1] then
+ envfrom = ef[1]['addr']
+ end
+
+ -- Envelope To
+ local envrcpt = 'test@example.com'
+ local rcpts = task:get_recipients();
+ if rcpts then
+ local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt)
+ return rcpt['addr']
+ end,
+ rcpts)), '\n')
+ if dcc_recipients then
+ envrcpt = dcc_recipients
+ end
+ end
+
+ -- Build the DCC query
+ -- https://www.dcc-servers.net/dcc/dcc-tree/dccifd.html#Protocol
+ local request_data = {
+ "header\n",
+ client .. "\n",
+ helo .. "\n",
+ envfrom .. "\n",
+ envrcpt .. "\n",
+ "\n",
+ content
+ }
+
+ local function dcc_callback(err, data, conn)
+
+ local function dcc_requery()
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule.timeout or 2.0,
+ upstream = upstream,
+ shutdown = true,
+ data = request_data,
+ callback = dcc_callback,
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ dcc_requery()
+
+ else
+ -- Parse the response
+ local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$")
+ lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"',
+ result, disposition, header)
+
+ if header then
+ -- Unfold header
+ header = header:gsub('\r?\n%s*', ' ')
+ local _, _, info = header:find("; (.-)$")
+ if (result == 'R') then
+ -- Reject
+ common.yield_result(task, rule, info, rule.default_score)
+ common.save_cache(task, digest, rule, info, rule.default_score)
+ elseif (result == 'T') then
+ -- Temporary failure
+ rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
+ dcc_requery()
+ elseif result == 'A' then
+
+ local opts = {}
+ local score = 0.0
+ info = info:lower()
+ local rep = info:match('rep=([^=%s]+)')
+
+ -- Adjust reputation if available
+ if rep then
+ rep = tonumber(rep)
+ end
+ if not rep then
+ rep = 1.0
+ end
+
+ local function check_threshold(what, num, lim)
+ local rnum
+ if num == 'many' then
+ rnum = lim
+ else
+ rnum = tonumber(num)
+ end
+
+ if rnum and rnum >= lim then
+ opts[#opts + 1] = string.format('%s=%s', what, num)
+ score = score + rep / 3.0
+ end
+ end
+
+ info = info:lower()
+ local body = info:match('body=([^=%s]+)')
+
+ if body then
+ check_threshold('body', body, rule.body_max)
+ end
+
+ local fuz1 = info:match('fuz1=([^=%s]+)')
+
+ if fuz1 then
+ check_threshold('fuz1', fuz1, rule.fuz1_max)
+ end
+
+ local fuz2 = info:match('fuz2=([^=%s]+)')
+
+ if fuz2 then
+ check_threshold('fuz2', fuz2, rule.fuz2_max)
+ end
+
+ if #opts > 0 and score > 0 then
+ task:insert_result(rule.symbol_bulk,
+ score,
+ opts)
+ common.save_cache(task, digest, rule, opts, score)
+ else
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+ rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result A - info: %s',
+ rule.log_prefix, info)
+ end
+ end
+ elseif result == 'G' then
+ -- do nothing
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result G - info: %s', rule.log_prefix, info)
+ end
+ elseif result == 'S' then
+ -- do nothing
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result S - info: %s', rule.log_prefix, info)
+ end
+ else
+ -- Unknown result
+ rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
+ common.yield_result(task, rule, 'error: ' .. result, 0.0, 'fail')
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ upstream = upstream,
+ data = request_data,
+ callback = dcc_callback,
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, dcc_check_uncached) then
+ return
+ else
+ dcc_check_uncached()
+ end
+
+end
+
+return {
+ type = { 'dcc', 'bulk', 'hash', 'scanner' },
+ description = 'dcc bulk scanner',
+ configure = dcc_config,
+ check = dcc_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
new file mode 100644
index 0000000..5a469c3
--- /dev/null
+++ b/lualib/lua_scanners/fprot.lua
@@ -0,0 +1,181 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module fprot
+-- This module contains fprot access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "fprot"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function fprot_config(opts)
+ local fprot_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 10200,
+ timeout = 5.0, -- FIXME: this will break task_timeout!
+ log_clean = false,
+ detection_category = "virus",
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ fprot_conf = lua_util.override_defaults(fprot_conf, opts)
+
+ if not fprot_conf.prefix then
+ fprot_conf.prefix = 'rs_' .. fprot_conf.name .. '_'
+ end
+
+ if not fprot_conf.log_prefix then
+ if fprot_conf.name:lower() == fprot_conf.type:lower() then
+ fprot_conf.log_prefix = fprot_conf.name
+ else
+ fprot_conf.log_prefix = fprot_conf.name .. ' (' .. fprot_conf.type .. ')'
+ end
+ end
+
+ if not fprot_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ fprot_conf['upstreams'] = upstream_list.create(rspamd_config,
+ fprot_conf['servers'],
+ fprot_conf.default_port)
+
+ if fprot_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', fprot_conf.name)
+ return fprot_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ fprot_conf['servers'])
+ return nil
+end
+
+local function fprot_check(task, content, digest, rule, maybe_part)
+ local function fprot_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local scan_id = task:get_queue_id()
+ if not scan_id then
+ scan_id = task:get_uid()
+ end
+ local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id,
+ #content)
+ local footer = '\n'
+
+ local function fprot_callback(err, data)
+ if err then
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ upstream:ok()
+ data = tostring(data)
+ local cached
+ local clean = string.match(data, '^0 <clean>')
+ if clean then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule['symbol'], rule['type'])
+ end
+ else
+ -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occurred
+ -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail
+ local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>')
+ if not vname then
+ rspamd_logger.errx(task, 'Unhandled response: %s', data)
+ else
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ fprot_check_uncached, maybe_part) then
+ return
+ else
+ fprot_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'fprot antivirus',
+ configure = fprot_config,
+ check = fprot_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua
new file mode 100644
index 0000000..682562d
--- /dev/null
+++ b/lualib/lua_scanners/icap.lua
@@ -0,0 +1,713 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+@module icap
+This module contains icap access functions.
+Currently tested with
+ - C-ICAP Squidclamav / echo
+ - Checkpoint Sandblast
+ - F-Secure Internet Gatekeeper
+ - Kaspersky Web Traffic Security
+ - Kaspersky Scan Engine 2.0
+ - McAfee Web Gateway 9/10/11
+ - Sophos Savdi
+ - Symantec (Rspamd <3.2, >=3.2 untested)
+ - Trend Micro IWSVA 6.0
+ - Trend Micro Web Gateway
+
+@TODO
+ - Preview / Continue
+ - Reqmod URL's
+ - Content-Type / Filename
+]] --
+
+--[[
+Configuration Notes:
+
+C-ICAP Squidclamav
+ scheme = "squidclamav";
+
+Checkpoint Sandblast example:
+ scheme = "sandblast";
+
+ESET Gateway Security / Antivirus for Linux example:
+ scheme = "scan";
+
+F-Secure Internet Gatekeeper example:
+ scheme = "respmod";
+ x_client_header = true;
+ x_rcpt_header = true;
+ x_from_header = true;
+
+Kaspersky Web Traffic Security example:
+ scheme = "av/respmod";
+ x_client_header = true;
+
+Kaspersky Web Traffic Security (as configured in kavicapd.xml):
+ scheme = "resp";
+ x_client_header = true;
+
+McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules)
+ scheme = "respmod";
+ x_client_header = true;
+
+Sophos SAVDI example:
+ # scheme as configured in savdi.conf (name option in service section)
+ scheme = "respmod";
+
+Symantec example:
+ scheme = "avscan";
+
+Trend Micro IWSVA example (X-Virus-ID/X-Infection-Found headers must be activated):
+ scheme = "avscan";
+ x_client_header = true;
+
+Trend Micro Web Gateway example (X-Virus-ID/X-Infection-Found headers must be activated):
+ scheme = "interscan";
+ x_client_header = true;
+]] --
+
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+local rspamd_util = require "rspamd_util"
+local rspamd_version = rspamd_version
+
+local N = 'icap'
+
+local function icap_config(opts)
+
+ local icap_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_all_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ scheme = "scan",
+ default_port = 1344,
+ ssl = false,
+ no_ssl_verify = false,
+ timeout = 10.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in one hour
+ message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ dynamic_scan = false,
+ user_agent = "Rspamd",
+ x_client_header = false,
+ x_rcpt_header = false,
+ x_from_header = false,
+ req_headers_enabled = true,
+ req_fake_url = "http://127.0.0.1/mail",
+ http_headers_enabled = true,
+ use_http_result_header = true,
+ use_http_3xx_as_threat = false,
+ use_specific_content_type = false, -- Use content type from a part where possible
+ }
+
+ icap_conf = lua_util.override_defaults(icap_conf, opts)
+
+ if not icap_conf.prefix then
+ icap_conf.prefix = 'rs_' .. icap_conf.name .. '_'
+ end
+
+ if not icap_conf.log_prefix then
+ icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+ end
+
+ if not icap_conf.log_prefix then
+ if icap_conf.name:lower() == icap_conf.type:lower() then
+ icap_conf.log_prefix = icap_conf.name
+ else
+ icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+ end
+ end
+
+ if not icap_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ icap_conf.upstreams = upstream_list.create(rspamd_config,
+ icap_conf.servers,
+ icap_conf.default_port)
+
+ if icap_conf.upstreams then
+ lua_util.add_debug_alias('external_services', icap_conf.name)
+ return icap_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ icap_conf.servers)
+ return nil
+end
+
+local function icap_check(task, content, digest, rule, maybe_part)
+ local function icap_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local http_headers = {}
+ local req_headers = {}
+ local tcp_options = {}
+ local threat_table = {}
+
+ -- Build extended User Agent
+ if rule.user_agent == "extended" then
+ rule.user_agent = string.format("Rspamd/%s-%s (%s/%s)",
+ rspamd_version('main'),
+ rspamd_version('id'),
+ rspamd_util.get_hostname(),
+ string.sub(task:get_uid(), 1, 6))
+ end
+
+ -- Build the icap queries
+ local options_request = {
+ string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(), rule.scheme),
+ string.format('Host: %s\r\n', addr:to_string()),
+ string.format("User-Agent: %s\r\n", rule.user_agent),
+ "Connection: keep-alive\r\n",
+ "Encapsulated: null-body=0\r\n\r\n",
+ }
+ if rule.user_agent == "none" then
+ table.remove(options_request, 3)
+ end
+
+ local respond_headers = {
+ -- Add main RESPMOD header before any other
+ string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme),
+ string.format('Host: %s\r\n', addr:to_string()),
+ }
+
+ local size = tonumber(#content)
+ local chunked_size = string.format("%x", size)
+
+ local function icap_callback(err, conn)
+
+ local function icap_requery(err_m, info)
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: %s Request Error: %s - retries left: %s',
+ rule.log_prefix, info, err_m, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp_options.host = addr:to_string()
+ tcp_options.port = addr:get_port()
+ tcp_options.callback = icap_callback
+ tcp_options.data = options_request
+ tcp_options.upstream = upstream
+
+ tcp.request(tcp_options)
+
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - error: %s', rule.log_prefix, err_m or '')
+ common.yield_result(task, rule, string.format('failed - error: %s', err_m),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ local function get_req_headers()
+
+ local in_client_ip = task:get_from_ip()
+ local req_hlen = 2
+ if maybe_part then
+ table.insert(req_headers,
+ string.format('GET http://%s/%s HTTP/1.0\r\n', in_client_ip, maybe_part:get_filename()))
+ if rule.use_specific_content_type then
+ table.insert(http_headers, string.format('Content-Type: %s/%s\r\n', maybe_part:get_detected_type()))
+ --else
+ -- To test: what content type is better for icap servers?
+ --table.insert(http_headers, 'Content-Type: text/plain\r\n')
+ end
+ else
+ table.insert(req_headers, string.format('GET %s HTTP/1.0\r\n', rule.req_fake_url))
+ table.insert(http_headers, string.format('Content-Type: application/octet-stream\r\n'))
+ end
+ table.insert(req_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
+ if rule.user_agent ~= "none" then
+ table.insert(req_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+
+ for _, h in ipairs(req_headers) do
+ req_hlen = req_hlen + tonumber(#h)
+ end
+
+ return req_hlen, req_headers
+
+ end
+
+ local function get_http_headers()
+ local http_hlen = 2
+ table.insert(http_headers, 'HTTP/1.0 200 OK\r\n')
+ table.insert(http_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
+ table.insert(http_headers, string.format('Server: %s\r\n', 'Apache/2.4'))
+ if rule.user_agent ~= "none" then
+ table.insert(http_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+ --table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html'))
+ table.insert(http_headers, string.format('Content-Length: %s\r\n', size))
+
+ for _, h in ipairs(http_headers) do
+ http_hlen = http_hlen + tonumber(#h)
+ end
+
+ return http_hlen, http_headers
+
+ end
+
+ local function get_respond_query()
+ local req_hlen = 0
+ local resp_req_headers
+ local http_hlen = 0
+ local resp_http_headers
+
+ -- Append all extra headers
+ if rule.user_agent ~= "none" then
+ table.insert(respond_headers,
+ string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+
+ if rule.req_headers_enabled then
+ req_hlen, resp_req_headers = get_req_headers()
+ end
+ if rule.http_headers_enabled then
+ http_hlen, resp_http_headers = get_http_headers()
+ end
+
+ if rule.req_headers_enabled and rule.http_headers_enabled then
+ local res_body_hlen = req_hlen + http_hlen
+ table.insert(respond_headers,
+ string.format('Encapsulated: req-hdr=0, res-hdr=%s, res-body=%s\r\n',
+ req_hlen, res_body_hlen))
+ elseif rule.http_headers_enabled then
+ table.insert(respond_headers,
+ string.format('Encapsulated: res-hdr=0, res-body=%s\r\n',
+ http_hlen))
+ else
+ table.insert(respond_headers, 'Encapsulated: res-body=0\r\n')
+ end
+
+ table.insert(respond_headers, '\r\n')
+ for _, h in ipairs(resp_req_headers) do
+ table.insert(respond_headers, h)
+ end
+ table.insert(respond_headers, '\r\n')
+ for _, h in ipairs(resp_http_headers) do
+ table.insert(respond_headers, h)
+ end
+ table.insert(respond_headers, '\r\n')
+ table.insert(respond_headers, chunked_size .. '\r\n')
+ table.insert(respond_headers, content)
+ table.insert(respond_headers, '\r\n0\r\n\r\n')
+ return respond_headers
+ end
+
+ local function add_respond_header(name, value)
+ if name and value then
+ table.insert(respond_headers, string.format('%s: %s\r\n', name, value))
+ end
+ end
+
+ local function result_header_table(result)
+ local icap_headers = {}
+ for s in result:gmatch("[^\r\n]+") do
+ if string.find(s, '^ICAP') then
+ icap_headers['icap'] = tostring(s)
+ elseif string.find(s, '^HTTP') then
+ icap_headers['http'] = tostring(s)
+ elseif string.find(s, '[%a%d-+]-:') then
+ local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)")
+ if key ~= nil then
+ icap_headers[key:lower()] = tostring(value)
+ end
+ end
+ end
+ lua_util.debugm(rule.name, task, '%s: icap_headers: %s',
+ rule.log_prefix, icap_headers)
+ return icap_headers
+ end
+
+ local function threat_table_add(icap_threat, maybe_split)
+
+ if maybe_split and string.find(icap_threat, ',') then
+ local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {}
+
+ for _, v in ipairs(threats) do
+ table.insert(threat_table, v)
+ end
+ else
+ table.insert(threat_table, icap_threat)
+ end
+ return true
+ end
+
+ local function icap_parse_result(headers)
+
+ --[[
+ @ToDo: handle type in response
+
+ Generic Strings:
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;
+
+ Symantec String:
+ icap: X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
+ icap: X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;
+
+ Sophos Strings:
+ icap: X-Virus-ID: Troj/DocDl-OYC
+ http: X-Blocked: Virus found during virus scan
+ http: X-Blocked-By: Sophos Anti-Virus
+
+ Kaspersky Web Traffic Security Strings:
+ icap: X-Virus-ID: HEUR:Backdoor.Java.QRat.gen
+ icap: X-Response-Info: blocked
+ icap: X-Virus-ID: no threats
+ icap: X-Response-Info: blocked
+ icap: X-Response-Info: passed
+ http: HTTP/1.1 403 Forbidden
+
+ Kaspersky Scan Engine 2.0 (ICAP mode)
+ icap: X-Virus-ID: EICAR-Test-File
+ http: HTTP/1.0 403 Forbidden
+
+ Trend Micro Strings:
+ icap: X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1;
+ http: HTTP/1.1 403 Forbidden (TMWS Blocked)
+ http: HTTP/1.1 403 Forbidden
+
+ F-Secure Internet Gatekeeper Strings:
+ icap: X-FSecure-Scan-Result: infected
+ icap: X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203"
+ icap: X-FSecure-Infected-Filename: "virus.doc"
+
+ ESET File Security for Linux 7.0
+ icap: X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA;
+ icap: X-Virus-ID: Trojaner
+ icap: X-Response-Info: Blocked
+
+ McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules)
+ icap: X-Virus-ID: EICAR test file
+ icap: X-Media-Type: text/plain
+ icap: X-Block-Result: 80
+ icap: X-Block-Reason: Malware found
+ icap: X-Block-Reason: Archive not supported
+ icap: X-Block-Reason: Media Type (Block List)
+ http: HTTP/1.0 403 VirusFound
+
+ C-ICAP Squidclamav
+ icap/http: X-Infection-Found: Type=0; Resolution=2; Threat={HEX}EICAR.TEST.3.UNOFFICIAL;
+ icap/http: X-Virus-ID: {HEX}EICAR.TEST.3.UNOFFICIAL
+ http: HTTP/1.0 307 Temporary Redirect
+ ]] --
+
+ -- Generic ICAP Headers
+ if headers['x-infection-found'] then
+ local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
+
+ -- Type=2 is typical for scan error returns
+ if icap_type and icap_type == '2' then
+ lua_util.debugm(rule.name, task,
+ '%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat)
+ common.yield_result(task, rule, icap_threat, 0,
+ 'fail', maybe_part)
+ return true
+ elseif icap_threat ~= nil then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat)
+ threat_table_add(icap_threat, false)
+ -- stupid workaround for unuseable x-infection-found header
+ -- but also x-virus-name set (McAfee Web Gateway 9)
+ elseif not icap_threat and headers['x-virus-name'] then
+ threat_table_add(headers['x-virus-name'], true)
+ else
+ threat_table_add(headers['x-infection-found'], true)
+ end
+ elseif headers['x-virus-name'] and headers['x-virus-name'] ~= "no threats" then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Virus-Name: %s', rule.log_prefix, headers['x-virus-name'])
+ threat_table_add(headers['x-virus-name'], true)
+ elseif headers['x-virus-id'] and headers['x-virus-id'] ~= "no threats" then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id'])
+ threat_table_add(headers['x-virus-id'], true)
+ -- FSecure X-Headers
+ elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then
+
+ local infected_filename = ""
+ local infection_name = "-unknown-"
+
+ if headers['x-fsecure-infected-filename'] then
+ infected_filename = string.gsub(headers['x-fsecure-infected-filename'], '[%s"]', '')
+ end
+ if headers['x-fsecure-infection-name'] then
+ infection_name = string.gsub(headers['x-fsecure-infection-name'], '[%s"]', '')
+ end
+
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-FSecure-Infection-Name (X-FSecure-Infected-Filename): %s (%s)',
+ rule.log_prefix, infection_name, infected_filename)
+
+ threat_table_add(infection_name, true)
+ -- McAfee Web Gateway manual extra headers
+ elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then
+ threat_table_add(headers['x-mwg-block-reason'], false)
+ -- Sophos SAVDI special http headers
+ elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then
+ threat_table_add(headers['x-blocked'], false)
+ elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then
+ threat_table_add(headers['x-block-reason'], false)
+ -- last try HTTP [4]xx return
+ elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then
+ threat_table_add(
+ string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ elseif rule.use_http_3xx_as_threat and
+ headers.http and
+ string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d')
+ then
+ threat_table_add(
+ string.format("pseudo-virus (redirect): %s",
+ string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ end
+
+ if #threat_table > 0 then
+ common.yield_result(task, rule, threat_table, rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, threat_table, rule.default_score, maybe_part)
+ return true
+ else
+ return false
+ end
+ end
+
+ local function icap_r_respond_http_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_respond_http_cb")
+ else
+ local result = tostring(data)
+
+ local icap_http_headers = result_header_table(result) or {}
+ -- Find HTTP/[12].x [234]xx response
+ if icap_http_headers.http and string.find(icap_http_headers.http, 'HTTP%/[12]%.. [234]%d%d') then
+ local icap_http_header_result = icap_parse_result(icap_http_headers)
+ if icap_http_header_result then
+ -- Threat found - close connection
+ connection:close()
+ else
+ common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
+ common.log_clean(task, rule)
+ end
+ else
+ rspamd_logger.errx(task, '%s: unhandled response |%s|',
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ end
+
+ local function icap_r_respond_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_respond_cb")
+ else
+ local result = tostring(data)
+
+ local icap_headers = result_header_table(result) or {}
+ -- Find ICAP/1.x 2xx response
+ if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+ local icap_header_result = icap_parse_result(icap_headers)
+ if icap_header_result then
+ -- Threat found - close connection
+ connection:close()
+ elseif not icap_header_result
+ and rule.use_http_result_header
+ and icap_headers.encapsulated
+ and not string.find(icap_headers.encapsulated, 'null%-body=0')
+ then
+ -- Try to read encapsulated HTTP Headers
+ lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers',
+ rule.log_prefix)
+ connection:add_read(icap_r_respond_http_cb, '\r\n\r\n')
+ else
+ connection:close()
+ common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
+ common.log_clean(task, rule)
+ end
+ elseif icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then
+ -- Find ICAP/1.x 5/4xx response
+ --[[
+ Symantec String:
+ ICAP/1.0 539 Aborted - No AV scanning license
+ SquidClamAV/C-ICAP:
+ ICAP/1.0 500 Server error
+ Eset:
+ ICAP/1.0 405 Forbidden
+ TrendMicro:
+ ICAP/1.0 400 Bad request
+ McAfee:
+ ICAP/1.0 418 Bad composition
+ ]]--
+ rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
+ common.yield_result(task, rule, icap_headers.icap, 0.0,
+ 'fail', maybe_part)
+ return false
+ else
+ rspamd_logger.errx(task, '%s: unhandled response |%s|',
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ end
+
+ local function icap_w_respond_cb(err_m, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_w_respond_cb")
+ else
+ connection:add_read(icap_r_respond_cb, '\r\n\r\n')
+ end
+ end
+
+ local function icap_r_options_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_options_cb")
+ else
+ local icap_headers = result_header_table(tostring(data))
+
+ if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+ if icap_headers['methods'] and string.find(icap_headers['methods'], 'RESPMOD') then
+ -- Allow "204 No Content" responses
+ -- https://datatracker.ietf.org/doc/html/rfc3507#section-4.6
+ if icap_headers['allow'] and string.find(icap_headers['allow'], '204') then
+ add_respond_header('Allow', '204')
+ end
+
+ if rule.x_client_header then
+ local client = task:get_from_ip()
+ if client then
+ add_respond_header('X-Client-IP', client:to_string())
+ end
+ end
+
+ -- F-Secure extra headers
+ if icap_headers['server'] and string.find(icap_headers['server'], 'f-secure icap server') then
+
+ if rule.x_rcpt_header then
+ local rcpt_to = task:get_principal_recipient()
+ if rcpt_to then
+ add_respond_header('X-Rcpt-To', rcpt_to)
+ end
+ end
+
+ if rule.x_from_header then
+ local mail_from = task:get_principal_recipient()
+ if mail_from and mail_from[1] then
+ add_respond_header('X-Rcpt-To', mail_from[1].addr)
+ end
+ end
+
+ end
+
+ if icap_headers.connection and icap_headers.connection:lower() == 'close' then
+ lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection',
+ rule.log_prefix, icap_headers.connection)
+ connection:close()
+ tcp_options.callback = icap_w_respond_cb
+ tcp_options.data = get_respond_query()
+ tcp.request(tcp_options)
+ else
+ connection:add_write(icap_w_respond_cb, get_respond_query())
+ end
+
+ else
+ rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
+ rule.log_prefix, icap_headers['methods'])
+ common.yield_result(task, rule, 'NO RESPMOD', 0.0,
+ 'fail', maybe_part)
+ end
+ else
+ rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
+ rule.log_prefix, icap_headers.icap or "-")
+ common.yield_result(task, rule, 'OPTIONS query failed', 0.0,
+ 'fail', maybe_part)
+ end
+ end
+ end
+
+ if err or conn == nil then
+ icap_requery(err, "options_request")
+ else
+ conn:add_read(icap_r_options_cb, '\r\n\r\n')
+ end
+ end
+
+ tcp_options.task = task
+ tcp_options.stop_pattern = '\r\n'
+ tcp_options.read = false
+ tcp_options.timeout = rule.timeout
+ tcp_options.callback = icap_callback
+ tcp_options.data = options_request
+
+ if rule.ssl then
+ tcp_options.ssl = true
+ if rule.no_ssl_verify then
+ tcp_options.no_ssl_verify = true
+ end
+ end
+
+ tcp_options.host = addr:to_string()
+ tcp_options.port = addr:get_port()
+ tcp_options.upstream = upstream
+
+ tcp.request(tcp_options)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ icap_check_uncached, maybe_part) then
+ return
+ else
+ icap_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'virus', 'virus', 'scanner' },
+ description = 'generic icap antivirus',
+ configure = icap_config,
+ check = icap_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua
new file mode 100644
index 0000000..e47cebe
--- /dev/null
+++ b/lualib/lua_scanners/init.lua
@@ -0,0 +1,75 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_scanners
+-- This module contains external scanners functions
+--]]
+
+local fun = require "fun"
+
+local exports = {
+}
+
+local function require_scanner(name)
+ local sc = require("lua_scanners/" .. name)
+
+ exports[sc.name or name] = sc
+end
+
+-- Antiviruses
+require_scanner('clamav')
+require_scanner('fprot')
+require_scanner('kaspersky_av')
+require_scanner('kaspersky_se')
+require_scanner('savapi')
+require_scanner('sophos')
+require_scanner('virustotal')
+require_scanner('avast')
+
+-- Other scanners
+require_scanner('dcc')
+require_scanner('oletools')
+require_scanner('icap')
+require_scanner('vadesecure')
+require_scanner('spamassassin')
+require_scanner('p0f')
+require_scanner('razor')
+require_scanner('pyzor')
+require_scanner('cloudmark')
+
+exports.add_scanner = function(name, t, conf_func, check_func)
+ assert(type(conf_func) == 'function' and type(check_func) == 'function',
+ 'bad arguments')
+ exports[name] = {
+ type = t,
+ configure = conf_func,
+ check = check_func,
+ }
+end
+
+exports.filter = function(t)
+ return fun.tomap(fun.filter(function(_, elt)
+ return type(elt) == 'table' and elt.type and (
+ (type(elt.type) == 'string' and elt.type == t) or
+ (type(elt.type) == 'table' and fun.any(function(tt)
+ return tt == t
+ end, elt.type))
+ )
+ end, exports))
+end
+
+return exports
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
new file mode 100644
index 0000000..d52cef0
--- /dev/null
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -0,0 +1,197 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module kaspersky
+-- This module contains kaspersky antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "kaspersky"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function kaspersky_config(opts)
+ local kaspersky_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ product_id = 0,
+ log_clean = false,
+ timeout = 5.0,
+ retransmits = 1, -- use local files, retransmits are useless
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ tmpdir = '/tmp',
+ }
+
+ kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts)
+
+ if not kaspersky_conf.prefix then
+ kaspersky_conf.prefix = 'rs_' .. kaspersky_conf.name .. '_'
+ end
+
+ if not kaspersky_conf.log_prefix then
+ if kaspersky_conf.name:lower() == kaspersky_conf.type:lower() then
+ kaspersky_conf.log_prefix = kaspersky_conf.name
+ else
+ kaspersky_conf.log_prefix = kaspersky_conf.name .. ' (' .. kaspersky_conf.type .. ')'
+ end
+ end
+
+ if not kaspersky_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ kaspersky_conf['upstreams'] = upstream_list.create(rspamd_config,
+ kaspersky_conf['servers'], 0)
+
+ if kaspersky_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', kaspersky_conf.name)
+ return kaspersky_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ kaspersky_conf['servers'])
+ return nil
+end
+
+local function kaspersky_check(task, content, digest, rule, maybe_part)
+ local function kaspersky_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+ local clamav_compat_cmd = string.format("nSCAN %s\n", fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for kaspersky scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure file cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ local function kaspersky_callback(err, data)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = kaspersky_callback,
+ data = { clamav_compat_cmd },
+ stop_pattern = '\n'
+ })
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule,
+ 'failed to scan and retransmits exceed', 0.0, 'fail',
+ maybe_part)
+ end
+
+ else
+ data = tostring(data)
+ local cached
+ lua_util.debugm(rule.name, task,
+ '%s [%s]: got reply: %s',
+ rule['symbol'], rule['type'], data)
+ if data == 'stream: OK' or data == fname .. ': OK' then
+ cached = 'OK'
+ common.log_clean(task, rule)
+ else
+ local vname = string.match(data, ': (.+) FOUND')
+ if vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ else
+ rspamd_logger.errx(task, 'unhandled response: %s', data)
+ common.yield_result(task, rule, 'unhandled response',
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = kaspersky_callback,
+ data = { clamav_compat_cmd },
+ stop_pattern = '\n'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ kaspersky_check_uncached, maybe_part) then
+ return
+ else
+ kaspersky_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'kaspersky antivirus',
+ configure = kaspersky_config,
+ check = kaspersky_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/kaspersky_se.lua b/lualib/lua_scanners/kaspersky_se.lua
new file mode 100644
index 0000000..5e0f2ea
--- /dev/null
+++ b/lualib/lua_scanners/kaspersky_se.lua
@@ -0,0 +1,287 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module kaspersky_se
+-- This module contains Kaspersky Scan Engine integration support
+-- https://www.kaspersky.com/scan-engine
+--]]
+
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'kaspersky_se'
+
+local function kaspersky_se_config(opts)
+
+ local default_conf = {
+ name = N,
+ default_port = 9999,
+ use_https = false,
+ use_files = false,
+ timeout = 5.0,
+ log_clean = false,
+ tmpdir = '/tmp',
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ }
+
+ default_conf = lua_util.override_defaults(default_conf, opts)
+
+ if not default_conf.prefix then
+ default_conf.prefix = 'rs_' .. default_conf.name .. '_'
+ end
+
+ if not default_conf.log_prefix then
+ if default_conf.name:lower() == default_conf.type:lower() then
+ default_conf.log_prefix = default_conf.name
+ else
+ default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
+ end
+ end
+
+ if not default_conf.servers and default_conf.socket then
+ default_conf.servers = default_conf.socket
+ end
+
+ if not default_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ default_conf.upstreams = upstream_list.create(rspamd_config,
+ default_conf.servers,
+ default_conf.default_port)
+
+ if default_conf.upstreams then
+ lua_util.add_debug_alias('external_services', default_conf.name)
+ return default_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ default_conf['servers'])
+ return nil
+end
+
+local function kaspersky_se_check(task, content, digest, rule, maybe_part)
+ local function kaspersky_se_check_uncached()
+ local function make_url(addr)
+ local url
+ local suffix = '/scanmemory'
+
+ if rule.use_files then
+ suffix = '/scanfile'
+ end
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ addr:get_port(), suffix)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ addr:get_port(), suffix)
+ end
+
+ return url
+ end
+
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = make_url(addr)
+ local hdrs = {
+ ['X-KAV-ProtocolVersion'] = '1',
+ ['X-KAV-Timeout'] = tostring(rule.timeout * 1000),
+ }
+
+ if task:has_from() then
+ hdrs['X-KAV-ObjectURL'] = string.format('[from:%s]', task:get_from()[1].addr)
+ end
+
+ local req_body
+
+ if rule.use_files then
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for kaspersky_se scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ req_body = fname
+ else
+ req_body = content
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = req_body,
+ headers = hdrs,
+ timeout = rule.timeout,
+ }
+
+ local function kas_callback(http_err, code, body, headers)
+
+ local function requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = make_url(addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+ request_data.upstream = upstream
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ end
+ end
+
+ if http_err then
+ requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ local data = string.gsub(tostring(body), '[\r\n%s]$', '')
+ local cached
+ lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
+ rule.log_prefix, data)
+
+ if data:find('^CLEAN') then
+ -- Handle CLEAN replies
+ if data == 'CLEAN' then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ end
+ elseif data == 'CLEAN AND CONTAINS OFFICE MACRO' then
+ common.yield_result(task, rule, 'File contains macros',
+ 0.0, 'macro', maybe_part)
+ cached = 'MACRO'
+ else
+ rspamd_logger.errx(task, '%s: unhandled clean response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ elseif data == 'SERVER_ERROR' then
+ rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'error:' .. data,
+ 0.0, 'fail', maybe_part)
+ elseif string.match(data, 'DETECT (.+)') then
+ local vname = string.match(data, 'DETECT (.+)')
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ elseif string.match(data, 'NON_SCANNED %((.+)%)') then
+ local why = string.match(data, 'NON_SCANNED %((.+)%)')
+
+ if why == 'PASSWORD PROTECTED' then
+ rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+ common.yield_result(task, rule, 'File is encrypted: ' .. why,
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ else
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+
+ end
+ end
+
+ request_data.callback = kas_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ kaspersky_se_check_uncached, maybe_part) then
+ return
+ else
+
+ kaspersky_se_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Kaspersky Scan Engine interface',
+ configure = kaspersky_se_config,
+ check = kaspersky_se_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
new file mode 100644
index 0000000..378e094
--- /dev/null
+++ b/lualib/lua_scanners/oletools.lua
@@ -0,0 +1,369 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module oletools
+-- This module contains oletools access functions.
+-- Olefy is needed: https://github.com/HeinleinSupport/olefy
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'oletools'
+
+local function oletools_config(opts)
+
+ local oletools_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 10050,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 86400, -- expire redis in 1d
+ min_size = 500,
+ symbol = "OLETOOLS",
+ message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
+ detection_category = "office macro",
+ default_score = 1,
+ action = false,
+ extended = false,
+ symbol_type = 'postfilter',
+ dynamic_scan = true,
+ }
+
+ oletools_conf = lua_util.override_defaults(oletools_conf, opts)
+
+ if not oletools_conf.prefix then
+ oletools_conf.prefix = 'rs_' .. oletools_conf.name .. '_'
+ end
+
+ if not oletools_conf.log_prefix then
+ if oletools_conf.name:lower() == oletools_conf.type:lower() then
+ oletools_conf.log_prefix = oletools_conf.name
+ else
+ oletools_conf.log_prefix = oletools_conf.name .. ' (' .. oletools_conf.type .. ')'
+ end
+ end
+
+ if not oletools_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ oletools_conf.upstreams = upstream_list.create(rspamd_config,
+ oletools_conf.servers,
+ oletools_conf.default_port)
+
+ if oletools_conf.upstreams then
+ lua_util.add_debug_alias('external_services', oletools_conf.name)
+ return oletools_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ oletools_conf.servers)
+ return nil
+end
+
+local function oletools_check(task, content, digest, rule, maybe_part)
+ local function oletools_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n'
+ local json_response = ""
+
+ local function oletools_callback(err, data, conn)
+
+ local function oletools_requery(error)
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = { protocol, content },
+ callback = oletools_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - err: %s', rule.log_prefix, error)
+ common.yield_result(task, rule,
+ 'failed to scan, maximum retransmits exceed - err: ' .. error,
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ if err then
+
+ oletools_requery(err)
+
+ else
+ json_response = json_response .. tostring(data)
+
+ if not string.find(json_response, '\t\n\n\t') and #data == 8192 then
+ lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s',
+ rule.log_prefix, #json_response, #data)
+ conn:add_read(oletools_callback)
+
+ else
+ local ucl_parser = ucl.parser()
+ local ok, ucl_err = ucl_parser:parse_string(tostring(json_response))
+ if not ok then
+ rspamd_logger.errx(task, "%s: error parsing json response, retry: %s",
+ rule.log_prefix, ucl_err)
+ oletools_requery(ucl_err)
+ return
+ end
+
+ local result = ucl_parser:get_object()
+
+ local oletools_rc = {
+ [0] = 'RETURN_OK',
+ [1] = 'RETURN_WARNINGS',
+ [2] = 'RETURN_WRONG_ARGS',
+ [3] = 'RETURN_FILE_NOT_FOUND',
+ [4] = 'RETURN_XGLOB_ERR',
+ [5] = 'RETURN_OPEN_ERROR',
+ [6] = 'RETURN_PARSE_ERROR',
+ [7] = 'RETURN_SEVERAL_ERRS',
+ [8] = 'RETURN_UNEXPECTED',
+ [9] = 'RETURN_ENCRYPTED',
+ }
+
+ -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
+ -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
+ -- Keep sorted to avoid dragons
+ local analysis_cat_table = {
+ autoexec = '-',
+ base64 = '-',
+ dridex = '-',
+ hex = '-',
+ iocs = '-',
+ macro_exist = '-',
+ suspicious = '-',
+ vba = '-'
+ }
+ local analysis_keyword_table = {}
+
+ for _, v in ipairs(result) do
+
+ if v.error ~= nil and v.type ~= 'error' then
+ -- olefy, not oletools error
+ rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
+ v.error)
+ if v.error == 'File too small' then
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'File too small to be scanned for macros')
+ return
+ else
+ oletools_requery(v.error)
+ end
+
+ elseif tostring(v.type) == "MetaInformation" and v.version ~= nil then
+ -- if MetaInformation section - check and print script and version
+
+ lua_util.debugm(N, task, '%s: version: %s %s', rule.log_prefix,
+ tostring(v.script_name), tostring(v.version))
+
+ elseif tostring(v.type) == "MetaInformation" and v.return_code ~= nil then
+ -- if MetaInformation section - check return_code
+
+ local oletools_rc_code = tonumber(v.return_code)
+ if oletools_rc_code == 9 then
+ rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'encrypted', maybe_part)
+ common.save_cache(task, digest, rule, 'encrypted', 1.0, maybe_part)
+ return
+ elseif oletools_rc_code == 5 then
+ rspamd_logger.warnx(task, '%s: olefy could not open the file - error: %s', rule.log_prefix,
+ result[2]['message'])
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'fail', maybe_part)
+ return
+ elseif oletools_rc_code > 6 then
+ rspamd_logger.errx(task, '%s: MetaInfo section error code: %s',
+ rule.log_prefix, oletools_rc[oletools_rc_code])
+ rspamd_logger.errx(task, '%s: MetaInfo section message: %s',
+ rule.log_prefix, result[2]['message'])
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'fail', maybe_part)
+ return
+ elseif oletools_rc_code > 1 then
+ rspamd_logger.errx(task, '%s: Error message: %s',
+ rule.log_prefix, result[2]['message'])
+ oletools_requery(oletools_rc[oletools_rc_code])
+ end
+
+ elseif tostring(v.type) == "error" then
+ -- error section found - check message
+ rspamd_logger.errx(task, '%s: Error section error code: %s',
+ rule.log_prefix, v.error)
+ rspamd_logger.errx(task, '%s: Error section message: %s',
+ rule.log_prefix, v.message)
+ --common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail')
+
+ elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then
+ -- analysis + macro found - evaluate response
+
+ if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then
+ rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
+ oletools_requery('oletools unhandled error')
+
+ elseif #v.macros > 0 then
+
+ analysis_cat_table.macro_exist = 'M'
+
+ lua_util.debugm(rule.name, task,
+ '%s: filename: %s', rule.log_prefix, result[2]['file'])
+ lua_util.debugm(rule.name, task,
+ '%s: type: %s', rule.log_prefix, result[2]['type'])
+
+ for _, m in ipairs(v.macros) do
+ lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' ..
+ 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
+ end
+
+ for _, a in ipairs(v.analysis) do
+ lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' ..
+ 'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
+ if a.type == 'AutoExec' then
+ analysis_cat_table.autoexec = 'A'
+ table.insert(analysis_keyword_table, a.keyword)
+ elseif a.type == 'Suspicious' then
+ if rule.extended == true or
+ (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
+ then
+ analysis_cat_table.suspicious = 'S'
+ table.insert(analysis_keyword_table, a.keyword)
+ end
+ elseif a.type == 'IOC' then
+ analysis_cat_table.iocs = 'I'
+ elseif a.type == 'Hex strings' then
+ analysis_cat_table.hex = 'H'
+ elseif a.type == 'Base64 strings' then
+ analysis_cat_table.base64 = 'B'
+ elseif a.type == 'Dridex strings' then
+ analysis_cat_table.dridex = 'D'
+ elseif a.type == 'VBA strings' then
+ analysis_cat_table.vba = 'V'
+ end
+ end
+ end
+ end
+ end
+
+ lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
+ lua_util.debugm(N, task, '%s: analysis_cat_table: %s', rule.log_prefix, analysis_cat_table)
+
+ if rule.extended == false and analysis_cat_table.autoexec == 'A' and analysis_cat_table.suspicious == 'S' then
+ -- use single string as virus name
+ local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
+ lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
+ common.yield_result(task, rule, threat, rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, threat, rule.default_score, maybe_part)
+
+ elseif rule.extended == true and #analysis_keyword_table > 0 then
+ -- report any flags (types) and any most keywords as individual virus name
+ local analysis_cat_table_values_sorted = {}
+
+ -- see https://github.com/rspamd/rspamd/commit/6bd3e2b9f49d1de3ab882aeca9c30bc7d526ac9d#commitcomment-40130493
+ -- for details
+ local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table)
+ table.sort(analysis_cat_table_keys_sorted)
+
+ for _, v in ipairs(analysis_cat_table_keys_sorted) do
+ table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v])
+ end
+
+ table.insert(analysis_keyword_table, 1, table.concat(analysis_cat_table_values_sorted))
+
+ lua_util.debugm(rule.name, task, '%s: extended threat result: %s',
+ rule.log_prefix, table.concat(analysis_keyword_table, ','))
+
+ common.yield_result(task, rule, analysis_keyword_table,
+ rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, analysis_keyword_table,
+ rule.default_score, maybe_part)
+
+ elseif analysis_cat_table.macro_exist == '-' and #analysis_keyword_table == 0 then
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'No macro found')
+
+ else
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'Scanned Macro is OK')
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = { protocol, content },
+ callback = oletools_callback,
+ })
+
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ oletools_check_uncached, maybe_part) then
+ return
+ else
+ oletools_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'attachment scanner', 'hash', 'scanner' },
+ description = 'oletools office macro scanner',
+ configure = oletools_config,
+ check = oletools_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua
new file mode 100644
index 0000000..7785f83
--- /dev/null
+++ b/lualib/lua_scanners/p0f.lua
@@ -0,0 +1,227 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module p0f
+-- This module contains p0f access functions
+--]]
+
+local tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local common = require "lua_scanners/common"
+
+-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317
+local S = {
+ BAD_QUERY = 0x0,
+ OK = 0x10,
+ NO_MATCH = 0x20
+}
+
+local N = 'p0f'
+
+local function p0f_check(task, ip, rule)
+
+ local function ip2bin(addr)
+ addr = addr:to_table()
+
+ for k, v in ipairs(addr) do
+ addr[k] = rspamd_util.pack('B', v)
+ end
+
+ return table.concat(addr)
+ end
+
+ local function trim(...)
+ local vars = { ... }
+
+ for k, v in ipairs(vars) do
+ -- skip numbers, trim only strings
+ if tonumber(vars[k]) == nil then
+ vars[k] = string.gsub(v, '[^%w-_\\.\\(\\) ]', '')
+ end
+ end
+
+ return lua_util.unpack(vars)
+ end
+
+ local function parse_p0f_response(data)
+ --[[
+ p0f_api_response[232]: magic, status, first_seen, last_seen, total_conn,
+ uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q,
+ os_name, os_flavor, http_name, http_flavor, link_type, language
+ ]]--
+
+ data = tostring(data)
+
+ -- API response must be 232 bytes long
+ if #data ~= 232 then
+ rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes',
+ rule.socket, #data)
+
+ common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket,
+ 0.0, 'fail')
+ return
+ end
+
+ local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name,
+ os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack(
+ 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data))
+
+ if status ~= S.OK then
+ if status == S.BAD_QUERY then
+ rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket)
+ common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket,
+ 0.0, 'fail')
+ end
+
+ return
+ end
+
+ local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor
+
+ task:get_mempool():set_variable('os_fingerprint', os_string, link_type,
+ uptime_min, distance)
+
+ if link_type and #link_type > 0 then
+ common.yield_result(task, rule, {
+ os_string,
+ 'link=' .. link_type,
+ 'distance=' .. distance },
+ 0.0)
+ else
+ common.yield_result(task, rule, {
+ os_string,
+ 'link=unknown',
+ 'distance=' .. distance },
+ 0.0)
+ end
+
+ return data
+ end
+
+ local function make_p0f_request()
+
+ local function check_p0f_cb(err, data)
+
+ local function redis_set_cb(redis_set_err)
+ if redis_set_err then
+ rspamd_logger.errx(task, 'redis received an error: %s', redis_set_err)
+ end
+ end
+
+ if err then
+ rspamd_logger.errx(task, 'p0f received an error: %s', err)
+ common.yield_result(task, rule, 'Error getting result: ' .. err,
+ 0.0, 'fail')
+ return
+ end
+
+ data = parse_p0f_response(data)
+
+ if rule.redis_params and data then
+ local key = rule.prefix .. ip:to_string()
+ local ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ true,
+ redis_set_cb,
+ 'SETEX',
+ { key, tostring(rule.expire), data }
+ )
+
+ if not ret then
+ rspamd_logger.warnx(task, 'error connecting to redis')
+ end
+ end
+ end
+
+ local query = rspamd_util.pack('I4 I1 c16', 0x50304601,
+ ip:get_version(), ip2bin(ip))
+
+ tcp.request({
+ host = rule.socket,
+ callback = check_p0f_cb,
+ data = { query },
+ task = task,
+ timeout = rule.timeout
+ })
+ end
+
+ local function redis_get_cb(err, data)
+ if err or type(data) ~= 'string' then
+ make_p0f_request()
+ else
+ parse_p0f_response(data)
+ end
+ end
+
+ local ret = nil
+ if rule.redis_params then
+ local key = rule.prefix .. ip:to_string()
+ ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ false,
+ redis_get_cb,
+ 'GET',
+ { key }
+ )
+ end
+
+ if not ret then
+ make_p0f_request() -- fallback to directly querying p0f
+ end
+end
+
+local function p0f_config(opts)
+ local p0f_conf = {
+ name = N,
+ timeout = 5,
+ symbol = 'P0F',
+ symbol_fail = 'P0F_FAIL',
+ patterns = {},
+ expire = 7200,
+ prefix = 'p0f',
+ detection_category = 'fingerprint',
+ message = '${SCANNER}: fingerprint matched: "${VIRUS}"'
+ }
+
+ p0f_conf = lua_util.override_defaults(p0f_conf, opts)
+ p0f_conf.patterns = common.create_regex_table(p0f_conf.patterns)
+
+ if not p0f_conf.log_prefix then
+ p0f_conf.log_prefix = p0f_conf.name
+ end
+
+ if not p0f_conf.socket then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+ return nil
+ end
+
+ return p0f_conf
+end
+
+return {
+ type = { N, 'fingerprint', 'scanner' },
+ description = 'passive OS fingerprinter',
+ configure = p0f_config,
+ check = p0f_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/pyzor.lua b/lualib/lua_scanners/pyzor.lua
new file mode 100644
index 0000000..75c1b4a
--- /dev/null
+++ b/lualib/lua_scanners/pyzor.lua
@@ -0,0 +1,206 @@
+--[[
+Copyright (c) 2021, defkev <defkev@gmail.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module pyzor
+-- This module contains pyzor access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'pyzor'
+local categories = { 'pyzor', 'bulk', 'hash', 'scanner' }
+
+local function pyzor_config(opts)
+
+ local pyzor_conf = {
+ text_part_min_words = 2,
+ default_port = 5953,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ detection_category = "hash",
+ cache_expire = 7200, -- expire redis in one hour
+ message = '${SCANNER}: Pyzor bulk message found: "${VIRUS}"',
+ default_score = 1.5,
+ action = false,
+ }
+
+ pyzor_conf = lua_util.override_defaults(pyzor_conf, opts)
+
+ if not pyzor_conf.prefix then
+ pyzor_conf.prefix = 'rext_' .. N .. '_'
+ end
+
+ if not pyzor_conf.log_prefix then
+ pyzor_conf.log_prefix = N .. ' (' .. pyzor_conf.detection_category .. ')'
+ end
+
+ if not pyzor_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ pyzor_conf['upstreams'] = upstream_list.create(rspamd_config,
+ pyzor_conf['servers'],
+ pyzor_conf.default_port)
+
+ if pyzor_conf['upstreams'] then
+ lua_util.add_debug_alias('external_services', N)
+ return pyzor_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ pyzor_conf['servers'])
+ return nil
+end
+
+local function pyzor_check(task, content, digest, rule)
+ local function pyzor_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local function pyzor_callback(err, data, conn)
+
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(N, task, '%s: retry IP: %s:%s err: %s',
+ rule.log_prefix, addr, addr:get_port(), err)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ shutdown = true,
+ data = content,
+ callback = pyzor_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0,
+ 'failed to scan and retransmits exceed')
+ end
+ else
+ -- pyzor output is unicode (\x09 -> tab, \0a -> newline)
+ -- public.pyzor.org:24441 (200, 'OK') 21285091 206759
+ -- server:port Code Diag Count WL-Count
+ local str_data = tostring(data)
+ lua_util.debugm(N, task, '%s: returned data: %s',
+ rule.log_prefix, str_data)
+ -- If pyzor would return JSON this wouldn't be necessary
+ local resp = {}
+ for v in string.gmatch(str_data, '[^\t]+') do
+ table.insert(resp, v)
+ end
+ -- rspamd_logger.infox(task, 'resp: %s', resp)
+ if resp[2] ~= [[(200, 'OK')]] then
+ rspamd_logger.errx(task, "error parsing response: %s", str_data)
+ return
+ end
+
+ local whitelisted = tonumber(resp[4])
+ local reported = tonumber(resp[3])
+
+ --rspamd_logger.infox(task, "%s - count=%s wl=%s", addr:to_string(), reported, whitelisted)
+
+ --[[
+ Weight is Count - WL-Count of rule.default_score in percent, e.g.
+ SPAM:
+ Count: 100 (100%)
+ WL-Count: 1 (1%)
+ rule.default_score: 1
+ Weight: 0.99
+ HAM:
+ Count: 10 (100%)
+ WL-Count: 10 (100%)
+ rule.default_score: 1
+ Weight: 0
+ ]]
+ local weight = tonumber(string.format("%.2f",
+ rule.default_score * (reported - whitelisted) / (reported + whitelisted)))
+ local info = string.format("count=%d wl=%d", reported, whitelisted)
+ local threat_string = string.format("bl_%d_wl_%d",
+ reported, whitelisted)
+
+ if weight > 0 then
+ lua_util.debugm(N, task, '%s: returned result is spam - info: %s',
+ rule.log_prefix, info)
+ common.yield_result(task, rule, threat_string, weight)
+ common.save_cache(task, digest, rule, threat_string, weight)
+ else
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result is ham - info: %s',
+ rule.log_prefix, info)
+ else
+ lua_util.debugm(N, task, '%s: returned result is ham - info: %s',
+ rule.log_prefix, info)
+ end
+ common.save_cache(task, digest, rule, 'OK', weight)
+ end
+
+ end
+ end
+
+ if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then
+ rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix)
+ return
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = content,
+ callback = pyzor_callback,
+ })
+ end
+ if common.condition_check_and_continue(task, content, rule, digest, pyzor_check_uncached) then
+ return
+ else
+ pyzor_check_uncached()
+ end
+end
+
+return {
+ type = categories,
+ description = 'pyzor bulk scanner',
+ configure = pyzor_config,
+ check = pyzor_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/razor.lua b/lualib/lua_scanners/razor.lua
new file mode 100644
index 0000000..fcc0a8e
--- /dev/null
+++ b/lualib/lua_scanners/razor.lua
@@ -0,0 +1,181 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module razor
+-- This module contains razor access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'razor'
+
+local function razor_config(opts)
+
+ local razor_conf = {
+ name = N,
+ default_port = 11342,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ dynamic_scan = false,
+ symbol_fail = 'RAZOR_FAIL',
+ symbol = 'RAZOR',
+ }
+
+ razor_conf = lua_util.override_defaults(razor_conf, opts)
+
+ if not razor_conf.prefix then
+ razor_conf.prefix = 'rs_' .. razor_conf.name .. '_'
+ end
+
+ if not razor_conf.log_prefix then
+ razor_conf.log_prefix = razor_conf.name
+ end
+
+ if not razor_conf.servers and razor_conf.socket then
+ razor_conf.servers = razor_conf.socket
+ end
+
+ if not razor_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ razor_conf.upstreams = upstream_list.create(rspamd_config,
+ razor_conf.servers,
+ razor_conf.default_port)
+
+ if razor_conf.upstreams then
+ lua_util.add_debug_alias('external_services', razor_conf.name)
+ return razor_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ razor_conf['servers'])
+ return nil
+end
+
+local function razor_check(task, content, digest, rule)
+ local function razor_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local function razor_callback(err, data, conn)
+
+ local function razor_requery()
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ data = content,
+ callback = razor_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ razor_requery()
+
+ else
+ --[[
+ @todo: Razorsocket currently only returns ham or spam. When the wrapper is fixed we should add dynamic scores here.
+ Maybe check spamassassin implementation.
+
+ This implementation is based on https://github.com/cgt/rspamd-plugins
+ Thanks @cgt!
+ ]] --
+
+ local threat_string = tostring(data)
+ if threat_string == "spam" then
+ lua_util.debugm(N, task, '%s: returned result is spam', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, threat_string, rule.default_score)
+ common.save_cache(task, digest, rule, threat_string, rule.default_score)
+ elseif threat_string == "ham" then
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: returned result is ham', rule['symbol'], rule['type'])
+ else
+ lua_util.debugm(N, task, '%s: returned result is ham', rule['symbol'], rule['type'])
+ end
+ common.save_cache(task, digest, rule, 'OK', rule.default_score)
+ else
+ rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string)
+ end
+
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ data = content,
+ callback = razor_callback,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, razor_check_uncached) then
+ return
+ else
+ razor_check_uncached()
+ end
+end
+
+return {
+ type = { 'razor', 'spam', 'hash', 'scanner' },
+ description = 'razor bulk scanner',
+ configure = razor_config,
+ check = razor_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
new file mode 100644
index 0000000..08f7b66
--- /dev/null
+++ b/lualib/lua_scanners/savapi.lua
@@ -0,0 +1,261 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module savapi
+-- This module contains avira savapi antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "savapi"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function savapi_config(opts)
+ local savapi_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 4444, -- note: You must set ListenAddress in savapi.conf
+ product_id = 0,
+ log_clean = false,
+ timeout = 15.0, -- FIXME: this will break task_timeout!
+ retransmits = 1, -- FIXME: useless, for local files
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ tmpdir = '/tmp',
+ }
+
+ savapi_conf = lua_util.override_defaults(savapi_conf, opts)
+
+ if not savapi_conf.prefix then
+ savapi_conf.prefix = 'rs_' .. savapi_conf.name .. '_'
+ end
+
+ if not savapi_conf.log_prefix then
+ if savapi_conf.name:lower() == savapi_conf.type:lower() then
+ savapi_conf.log_prefix = savapi_conf.name
+ else
+ savapi_conf.log_prefix = savapi_conf.name .. ' (' .. savapi_conf.type .. ')'
+ end
+ end
+
+ if not savapi_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ savapi_conf['upstreams'] = upstream_list.create(rspamd_config,
+ savapi_conf['servers'],
+ savapi_conf.default_port)
+
+ if savapi_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', savapi_conf.name)
+ return savapi_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ savapi_conf['servers'])
+ return nil
+end
+
+local function savapi_check(task, content, digest, rule)
+ local function savapi_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for savapi scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ local vnames = {}
+
+ -- Forward declaration for recursive calls
+ local savapi_scan1_cb
+
+ local function savapi_fin_cb(err, conn)
+ local vnames_reordered = {}
+ -- Swap table
+ for virus, _ in pairs(vnames) do
+ table.insert(vnames_reordered, virus)
+ end
+ lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
+ if #vnames_reordered > 0 then
+ local vname = {}
+ for _, virus in ipairs(vnames_reordered) do
+ table.insert(vname, virus)
+ end
+
+ common.yield_result(task, rule, vname)
+ common.save_cache(task, digest, rule, vname)
+ end
+ if conn then
+ conn:close()
+ end
+ end
+
+ local function savapi_scan2_cb(err, data, conn)
+ local result = tostring(data)
+ lua_util.debugm(rule.name, task, "%s: got reply: %s",
+ rule.type, result)
+
+ -- Terminal response - clean
+ if string.find(result, '200') or string.find(result, '210') then
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
+ end
+ common.save_cache(task, digest, rule, 'OK')
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+
+ -- Terminal response - infected
+ elseif string.find(result, '319') then
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+
+ -- Non-terminal response
+ elseif string.find(result, '310') then
+ local virus
+ virus = result:match "310.*<<<%s(.*)%s+;.*;.*"
+ if not virus then
+ virus = result:match "310%s(.*)%s+;.*;.*"
+ if not virus then
+ rspamd_logger.errx(task, "%s: virus result unparseable: %s",
+ rule['type'], result)
+ common.yield_result(task, rule, 'virus result unparseable: ' .. result, 0.0, 'fail')
+ return
+ end
+ end
+ -- Store unique virus names
+ vnames[virus] = 1
+ -- More content is expected
+ conn:add_write(savapi_scan1_cb, '\n')
+ end
+ end
+
+ savapi_scan1_cb = function(err, conn)
+ conn:add_read(savapi_scan2_cb, '\n')
+ end
+
+ -- 100 PRODUCT:xyz
+ local function savapi_greet2_cb(err, data, conn)
+ local result = tostring(data)
+ if string.find(result, '100 PRODUCT') then
+ lua_util.debugm(rule.name, task, "%s: scanning file: %s",
+ rule['type'], fname)
+ conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n',
+ fname) })
+ else
+ rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'],
+ rule['product_id'])
+ common.yield_result(task, rule, 'invalid product id: ' .. result, 0.0, 'fail')
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+ end
+ end
+
+ local function savapi_greet1_cb(err, conn)
+ conn:add_read(savapi_greet2_cb, '\n')
+ end
+
+ local function savapi_callback_init(err, data, conn)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = savapi_callback_init,
+ stop_pattern = { '\n' },
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ else
+ local result = tostring(data)
+
+ -- 100 SAVAPI:4.0 greeting
+ if string.find(result, '100') then
+ conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) })
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = savapi_callback_init,
+ stop_pattern = { '\n' },
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, savapi_check_uncached) then
+ return
+ else
+ savapi_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'savapi avira antivirus',
+ configure = savapi_config,
+ check = savapi_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
new file mode 100644
index 0000000..d9b64f1
--- /dev/null
+++ b/lualib/lua_scanners/sophos.lua
@@ -0,0 +1,192 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module savapi
+-- This module contains avira savapi antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "sophos"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function sophos_config(opts)
+ local sophos_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 4010,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ }
+
+ sophos_conf = lua_util.override_defaults(sophos_conf, opts)
+
+ if not sophos_conf.prefix then
+ sophos_conf.prefix = 'rs_' .. sophos_conf.name .. '_'
+ end
+
+ if not sophos_conf.log_prefix then
+ if sophos_conf.name:lower() == sophos_conf.type:lower() then
+ sophos_conf.log_prefix = sophos_conf.name
+ else
+ sophos_conf.log_prefix = sophos_conf.name .. ' (' .. sophos_conf.type .. ')'
+ end
+ end
+
+ if not sophos_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ sophos_conf['upstreams'] = upstream_list.create(rspamd_config,
+ sophos_conf['servers'],
+ sophos_conf.default_port)
+
+ if sophos_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', sophos_conf.name)
+ return sophos_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ sophos_conf['servers'])
+ return nil
+end
+
+local function sophos_check(task, content, digest, rule, maybe_part)
+ local function sophos_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local protocol = 'SSSP/1.0\n'
+ local streamsize = string.format('SCANDATA %d\n', #content)
+ local bye = 'BYE\n'
+
+ local function sophos_callback(err, data, conn)
+
+ if err then
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = sophos_callback,
+ data = { protocol, streamsize, content, bye }
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ data = tostring(data)
+ lua_util.debugm(rule.name, task,
+ '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+ local vname = string.match(data, 'VIRUS (%S+) ')
+ local cached
+ if vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ common.save_cache(task, digest, rule, vname, 1.0, maybe_part)
+ else
+ if string.find(data, 'DONE OK') then
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task,
+ '%s: message or mime_part is clean', rule.log_prefix)
+ end
+ cached = 'OK'
+ -- not finished - continue
+ elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
+ conn:add_read(sophos_callback)
+ elseif string.find(data, 'FAIL 0212') then
+ rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)',
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ elseif string.find(data, 'REJ 4') then
+ rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)',
+ 0.0, 'fail', maybe_part)
+ -- explicitly set REJ1 message when SAVDIreports a protocol error
+ elseif string.find(data, 'REJ 1') then
+ rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)',
+ 0.0, 'fail', maybe_part)
+ else
+ rspamd_logger.errx(task, 'unhandled response: %s', data)
+ common.yield_result(task, rule, 'unhandled response: ' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = sophos_callback,
+ data = { protocol, streamsize, content, bye }
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ sophos_check_uncached, maybe_part) then
+ return
+ else
+ sophos_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'sophos antivirus',
+ configure = sophos_config,
+ check = sophos_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua
new file mode 100644
index 0000000..f425924
--- /dev/null
+++ b/lualib/lua_scanners/spamassassin.lua
@@ -0,0 +1,213 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module spamassassin
+-- This module contains spamd access functions.
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'spamassassin'
+
+local function spamassassin_config(opts)
+
+ local spamassassin_conf = {
+ N = N,
+ scan_mime_parts = false,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 783,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ symbol = "SPAMD",
+ message = '${SCANNER}: Spamassassin bulk message found: "${VIRUS}"',
+ detection_category = "spam",
+ default_score = 1,
+ action = false,
+ extended = false,
+ symbol_type = 'postfilter',
+ dynamic_scan = true,
+ }
+
+ spamassassin_conf = lua_util.override_defaults(spamassassin_conf, opts)
+
+ if not spamassassin_conf.prefix then
+ spamassassin_conf.prefix = 'rs_' .. spamassassin_conf.name .. '_'
+ end
+
+ if not spamassassin_conf.log_prefix then
+ if spamassassin_conf.name:lower() == spamassassin_conf.type:lower() then
+ spamassassin_conf.log_prefix = spamassassin_conf.name
+ else
+ spamassassin_conf.log_prefix = spamassassin_conf.name .. ' (' .. spamassassin_conf.type .. ')'
+ end
+ end
+
+ if not spamassassin_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ spamassassin_conf.upstreams = upstream_list.create(rspamd_config,
+ spamassassin_conf.servers,
+ spamassassin_conf.default_port)
+
+ if spamassassin_conf.upstreams then
+ lua_util.add_debug_alias('external_services', spamassassin_conf.N)
+ return spamassassin_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ spamassassin_conf.servers)
+ return nil
+end
+
+local function spamassassin_check(task, content, digest, rule)
+ local function spamassassin_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ -- Build the spamd query
+ -- https://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+ local request_data = {
+ "HEADERS SPAMC/1.5\r\n",
+ "User: root\r\n",
+ "Content-length: " .. #content .. "\r\n",
+ "\r\n",
+ content,
+ }
+
+ local function spamassassin_callback(err, data)
+
+ local function spamassassin_requery(error)
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, error, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ data = request_data,
+ callback = spamassassin_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - err: %s', rule.log_prefix, error)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ spamassassin_requery(err)
+
+ else
+ --lua_util.debugm(rule.N, task, '%s: returned result: %s', rule.log_prefix, data)
+
+ --[[
+ patterns tested against Spamassassin 3.4.6
+
+ X-Spam-Status: No, score=1.1 required=5.0 tests=HTML_MESSAGE,MIME_HTML_ONLY,
+ TVD_RCVD_SPACE_BRACKET,UNPARSEABLE_RELAY autolearn=no
+ autolearn_force=no version=3.4.6
+ ]] --
+ local header = string.gsub(tostring(data), "[\r\n]+[\t ]", " ")
+ --lua_util.debugm(rule.N, task, '%s: returned header: %s', rule.log_prefix, header)
+
+ local symbols = ""
+ local spam_score = 0
+ for s in header:gmatch("[^\r\n]+") do
+ if string.find(s, 'X%-Spam%-Status: %S+, score') then
+ local pattern_symbols = "X%-Spam%-Status: %S+, score%=([%-%d%.]+)%s.*tests%=(.*,?)(%s*%S+)%sautolearn.*"
+ spam_score = string.gsub(s, pattern_symbols, "%1")
+ symbols = string.gsub(s, pattern_symbols, "%2%3")
+ symbols = string.gsub(symbols, "%s", "")
+ end
+ end
+
+ lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|',
+ rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score))
+
+ if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then
+
+ if rule.extended == false then
+ common.yield_result(task, rule, symbols, spam_score)
+ common.save_cache(task, digest, rule, symbols, spam_score)
+ else
+ local symbols_table = lua_util.str_split(symbols, ",")
+ lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
+
+ common.yield_result(task, rule, symbols_table, spam_score)
+ common.save_cache(task, digest, rule, symbols_table, spam_score)
+ end
+ else
+ common.save_cache(task, digest, rule, 'OK')
+ common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ data = request_data,
+ callback = spamassassin_callback,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, spamassassin_check_uncached) then
+ return
+ else
+ spamassassin_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'spam', 'scanner' },
+ description = 'spamassassin spam scanner',
+ configure = spamassassin_config,
+ check = spamassassin_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua
new file mode 100644
index 0000000..826573a
--- /dev/null
+++ b/lualib/lua_scanners/vadesecure.lua
@@ -0,0 +1,351 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module vadesecure
+-- This module contains Vadesecure Filterd interface
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'vadesecure'
+
+local function vade_config(opts)
+
+ local vade_conf = {
+ name = N,
+ default_port = 23808,
+ url = '/api/v1/scan',
+ use_https = false,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ log_spamcause = true,
+ symbol_fail = 'VADE_FAIL',
+ symbol = 'VADE_CHECK',
+ settings_outbound = nil, -- Set when there is a settings id for outbound messages
+ symbols = {
+ clean = {
+ symbol = 'VADE_CLEAN',
+ score = -0.5,
+ description = 'VadeSecure decided message to be clean'
+ },
+ spam = {
+ high = {
+ symbol = 'VADE_SPAM_HIGH',
+ score = 8.0,
+ description = 'VadeSecure decided message to be clearly spam'
+ },
+ medium = {
+ symbol = 'VADE_SPAM_MEDIUM',
+ score = 5.0,
+ description = 'VadeSecure decided message to be highly likely spam'
+ },
+ low = {
+ symbol = 'VADE_SPAM_LOW',
+ score = 2.0,
+ description = 'VadeSecure decided message to be likely spam'
+ },
+ },
+ malware = {
+ symbol = 'VADE_MALWARE',
+ score = 8.0,
+ description = 'VadeSecure decided message to be malware'
+ },
+ scam = {
+ symbol = 'VADE_SCAM',
+ score = 7.0,
+ description = 'VadeSecure decided message to be scam'
+ },
+ phishing = {
+ symbol = 'VADE_PHISHING',
+ score = 8.0,
+ description = 'VadeSecure decided message to be phishing'
+ },
+ commercial = {
+ symbol = 'VADE_COMMERCIAL',
+ score = 0.0,
+ description = 'VadeSecure decided message to be commercial message'
+ },
+ community = {
+ symbol = 'VADE_COMMUNITY',
+ score = 0.0,
+ description = 'VadeSecure decided message to be community message'
+ },
+ transactional = {
+ symbol = 'VADE_TRANSACTIONAL',
+ score = 0.0,
+ description = 'VadeSecure decided message to be transactional message'
+ },
+ suspect = {
+ symbol = 'VADE_SUSPECT',
+ score = 3.0,
+ description = 'VadeSecure decided message to be suspicious message'
+ },
+ bounce = {
+ symbol = 'VADE_BOUNCE',
+ score = 0.0,
+ description = 'VadeSecure decided message to be bounce message'
+ },
+ other = 'VADE_OTHER',
+ }
+ }
+
+ vade_conf = lua_util.override_defaults(vade_conf, opts)
+
+ if not vade_conf.prefix then
+ vade_conf.prefix = 'rs_' .. vade_conf.name .. '_'
+ end
+
+ if not vade_conf.log_prefix then
+ if vade_conf.name:lower() == vade_conf.type:lower() then
+ vade_conf.log_prefix = vade_conf.name
+ else
+ vade_conf.log_prefix = vade_conf.name .. ' (' .. vade_conf.type .. ')'
+ end
+ end
+
+ if not vade_conf.servers and vade_conf.socket then
+ vade_conf.servers = vade_conf.socket
+ end
+
+ if not vade_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ vade_conf.upstreams = upstream_list.create(rspamd_config,
+ vade_conf.servers,
+ vade_conf.default_port)
+
+ if vade_conf.upstreams then
+ lua_util.add_debug_alias('external_services', vade_conf.name)
+ return vade_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ vade_conf['servers'])
+ return nil
+end
+
+local function vade_check(task, content, digest, rule, maybe_part)
+ local function vade_check_uncached()
+ local function vade_url(addr)
+ local url
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ rule.default_port, rule.url)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ rule.default_port, rule.url)
+ end
+
+ return url
+ end
+
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = vade_url(addr)
+ local hdrs = {}
+
+ local helo = task:get_helo()
+ if helo then
+ hdrs['X-Helo'] = helo
+ end
+ local mail_from = task:get_from('smtp') or {}
+ if mail_from[1] and #mail_from[1].addr > 1 then
+ hdrs['X-Mailfrom'] = mail_from[1].addr
+ end
+
+ local rcpt_to = task:get_recipients('smtp')
+ if rcpt_to then
+ hdrs['X-Rcptto'] = {}
+ for _, r in ipairs(rcpt_to) do
+ table.insert(hdrs['X-Rcptto'], r.addr)
+ end
+ end
+
+ local fip = task:get_from_ip()
+ if fip and fip:is_valid() then
+ hdrs['X-Inet'] = tostring(fip)
+ end
+
+ if rule.settings_outbound then
+ local settings_id = task:get_settings_id()
+
+ if settings_id then
+ local lua_settings = require "lua_settings"
+ -- Convert to string
+ settings_id = lua_settings.settings_by_id(settings_id)
+
+ if settings_id then
+ settings_id = settings_id.name or ''
+
+ if settings_id == rule.settings_outbound then
+ lua_util.debugm(rule.name, task, '%s settings has matched outbound',
+ settings_id)
+ hdrs['X-Params'] = 'mode=smtpout'
+ end
+ end
+ end
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = task:get_content(),
+ headers = hdrs,
+ timeout = rule.timeout,
+ }
+
+ local function vade_callback(http_err, code, body, headers)
+
+ local function vade_requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = vade_url(addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ end
+ end
+
+ if http_err then
+ vade_requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
+ return
+ end
+ local obj = parser:get_object()
+ local verdict = obj.verdict
+ if not verdict then
+ rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj)
+ task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict')
+ return
+ end
+ local vparts = lua_util.str_split(verdict, ":")
+ verdict = table.remove(vparts, 1) or verdict
+
+ local sym = rule.symbols[verdict]
+ if not sym then
+ sym = rule.symbols.other
+ end
+
+ if not sym.symbol then
+ -- Subcategory match
+ local lvl = 'low'
+ if vparts and vparts[1] then
+ lvl = vparts[1]
+ end
+
+ if sym[lvl] then
+ sym = sym[lvl]
+ else
+ sym = rule.symbols.other
+ end
+ end
+
+ local opts = {}
+ if obj.score then
+ table.insert(opts, 'score=' .. obj.score)
+ end
+ if obj.elapsed then
+ table.insert(opts, 'elapsed=' .. obj.elapsed)
+ end
+
+ if rule.log_spamcause and obj.spamcause then
+ rspamd_logger.infox(task, 'vadesecure verdict="%s", score=%s, spamcause="%s", message-id="%s"',
+ verdict, obj.score, obj.spamcause, task:get_message_id())
+ else
+ lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"',
+ verdict, obj.score, obj.spamcause)
+ end
+
+ if #vparts > 0 then
+ table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':'))
+ end
+
+ task:insert_result(sym.symbol, 1.0, opts)
+ end
+ end
+
+ request_data.callback = vade_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ vade_check_uncached, maybe_part) then
+ return
+ else
+ vade_check_uncached()
+ end
+
+end
+
+return {
+ type = { 'vadesecure', 'scanner' },
+ description = 'VadeSecure Filterd interface',
+ configure = vade_config,
+ check = vade_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua
new file mode 100644
index 0000000..d937c41
--- /dev/null
+++ b/lualib/lua_scanners/virustotal.lua
@@ -0,0 +1,214 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module virustotal
+-- This module contains Virustotal integration support
+-- https://www.virustotal.com/
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'virustotal'
+
+local function virustotal_config(opts)
+
+ local default_conf = {
+ name = N,
+ url = 'https://www.virustotal.com/vtapi/v2/file',
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ apikey = nil, -- Required to set by user
+ -- Specific for virustotal
+ minimum_engines = 3, -- Minimum required to get scored
+ full_score_engines = 7, -- After this number we set max score
+ }
+
+ default_conf = lua_util.override_defaults(default_conf, opts)
+
+ if not default_conf.prefix then
+ default_conf.prefix = 'rs_' .. default_conf.name .. '_'
+ end
+
+ if not default_conf.log_prefix then
+ if default_conf.name:lower() == default_conf.type:lower() then
+ default_conf.log_prefix = default_conf.name
+ else
+ default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
+ end
+ end
+
+ if not default_conf.apikey then
+ rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks')
+
+ return nil
+ end
+
+ lua_util.add_debug_alias('external_services', default_conf.name)
+ return default_conf
+end
+
+local function virustotal_check(task, content, digest, rule, maybe_part)
+ local function virustotal_check_uncached()
+ local function make_url(hash)
+ return string.format('%s/report?apikey=%s&resource=%s',
+ rule.url, rule.apikey, hash)
+ end
+
+ local hash = rspamd_cryptobox_hash.create_specific('md5')
+ hash:update(content)
+ hash = hash:hex()
+
+ local url = make_url(hash)
+ lua_util.debugm(N, task, "send request %s", url)
+ local request_data = {
+ task = task,
+ url = url,
+ timeout = rule.timeout,
+ }
+
+ local function vt_http_callback(http_err, code, body, headers)
+ if http_err then
+ rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers)
+ else
+ local cached
+ local dyn_score
+ -- Parse the response
+ if code ~= 200 then
+ if code == 404 then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ else
+ lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ end
+ elseif code == 204 then
+ -- Request rate limit exceeded
+ rspamd_logger.infox(task, 'virustotal request rate limit exceeded')
+ task:insert_result(rule.symbol_fail, 1.0, 'rate limit exceeded')
+ return
+ else
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ else
+ local ucl = require "ucl"
+ local parser = ucl.parser()
+ local res, json_err = parser:parse_string(body)
+
+ lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
+ rule.log_prefix, body)
+
+ if res then
+ local obj = parser:get_object()
+ if not obj.positives or type(obj.positives) ~= 'number' then
+ if obj.response_code then
+ if obj.response_code == 0 then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ else
+ lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ end
+ else
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ 'bad response code: ' .. tostring(obj.response_code), body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
+ return
+ end
+ else
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ 'no response_code', body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
+ return
+ end
+ else
+ if obj.positives < rule.minimum_engines then
+ lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min',
+ rule.log_prefix, obj.positives, rule.minimum_engines)
+ -- TODO: add proper hashing!
+ cached = 'OK'
+ else
+ if obj.positives > rule.full_score_engines then
+ dyn_score = 1.0
+ else
+ local norm_pos = obj.positives - rule.minimum_engines
+ dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines)
+ end
+
+ if dyn_score < 0 or dyn_score > 1 then
+ dyn_score = 1.0
+ end
+ local sopt = string.format("%s:%s/%s",
+ hash, obj.positives, obj.total)
+ common.yield_result(task, rule, sopt, dyn_score, nil, maybe_part)
+ cached = sopt
+ end
+ end
+ else
+ -- not res
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ json_err, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err)
+ return
+ end
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, dyn_score, maybe_part)
+ end
+ end
+ end
+
+ request_data.callback = vt_http_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ virustotal_check_uncached) then
+ return
+ else
+
+ virustotal_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Virustotal integration',
+ configure = virustotal_config,
+ check = virustotal_check,
+ name = N
+}