diff options
Diffstat (limited to 'lualib/lua_scanners/icap.lua')
| -rw-r--r-- | lualib/lua_scanners/icap.lua | 713 |
1 files changed, 713 insertions, 0 deletions
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua new file mode 100644 index 0000000..682562d --- /dev/null +++ b/lualib/lua_scanners/icap.lua @@ -0,0 +1,713 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> +Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[ +@module icap +This module contains icap access functions. +Currently tested with + - C-ICAP Squidclamav / echo + - Checkpoint Sandblast + - F-Secure Internet Gatekeeper + - Kaspersky Web Traffic Security + - Kaspersky Scan Engine 2.0 + - McAfee Web Gateway 9/10/11 + - Sophos Savdi + - Symantec (Rspamd <3.2, >=3.2 untested) + - Trend Micro IWSVA 6.0 + - Trend Micro Web Gateway + +@TODO + - Preview / Continue + - Reqmod URL's + - Content-Type / Filename +]] -- + +--[[ +Configuration Notes: + +C-ICAP Squidclamav + scheme = "squidclamav"; + +Checkpoint Sandblast example: + scheme = "sandblast"; + +ESET Gateway Security / Antivirus for Linux example: + scheme = "scan"; + +F-Secure Internet Gatekeeper example: + scheme = "respmod"; + x_client_header = true; + x_rcpt_header = true; + x_from_header = true; + +Kaspersky Web Traffic Security example: + scheme = "av/respmod"; + x_client_header = true; + +Kaspersky Web Traffic Security (as configured in kavicapd.xml): + scheme = "resp"; + x_client_header = true; + +McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules) + scheme = "respmod"; + x_client_header = true; + +Sophos SAVDI example: + # scheme as configured in savdi.conf (name option in service section) + scheme = "respmod"; + +Symantec example: + scheme = "avscan"; + +Trend Micro IWSVA example (X-Virus-ID/X-Infection-Found headers must be activated): + scheme = "avscan"; + x_client_header = true; + +Trend Micro Web Gateway example (X-Virus-ID/X-Infection-Found headers must be activated): + scheme = "interscan"; + x_client_header = true; +]] -- + + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_logger = require "rspamd_logger" +local common = require "lua_scanners/common" +local rspamd_util = require "rspamd_util" +local rspamd_version = rspamd_version + +local N = 'icap' + +local function icap_config(opts) + + local icap_conf = { + name = N, + scan_mime_parts = true, + scan_all_mime_parts = true, + scan_text_mime = false, + scan_image_mime = false, + scheme = "scan", + default_port = 1344, + ssl = false, + no_ssl_verify = false, + timeout = 10.0, + log_clean = false, + retransmits = 2, + cache_expire = 7200, -- expire redis in one hour + message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"', + detection_category = "virus", + default_score = 1, + action = false, + dynamic_scan = false, + user_agent = "Rspamd", + x_client_header = false, + x_rcpt_header = false, + x_from_header = false, + req_headers_enabled = true, + req_fake_url = "http://127.0.0.1/mail", + http_headers_enabled = true, + use_http_result_header = true, + use_http_3xx_as_threat = false, + use_specific_content_type = false, -- Use content type from a part where possible + } + + icap_conf = lua_util.override_defaults(icap_conf, opts) + + if not icap_conf.prefix then + icap_conf.prefix = 'rs_' .. icap_conf.name .. '_' + end + + if not icap_conf.log_prefix then + icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')' + end + + if not icap_conf.log_prefix then + if icap_conf.name:lower() == icap_conf.type:lower() then + icap_conf.log_prefix = icap_conf.name + else + icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')' + end + end + + if not icap_conf.servers then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + icap_conf.upstreams = upstream_list.create(rspamd_config, + icap_conf.servers, + icap_conf.default_port) + + if icap_conf.upstreams then + lua_util.add_debug_alias('external_services', icap_conf.name) + return icap_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + icap_conf.servers) + return nil +end + +local function icap_check(task, content, digest, rule, maybe_part) + local function icap_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local http_headers = {} + local req_headers = {} + local tcp_options = {} + local threat_table = {} + + -- Build extended User Agent + if rule.user_agent == "extended" then + rule.user_agent = string.format("Rspamd/%s-%s (%s/%s)", + rspamd_version('main'), + rspamd_version('id'), + rspamd_util.get_hostname(), + string.sub(task:get_uid(), 1, 6)) + end + + -- Build the icap queries + local options_request = { + string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), + string.format("User-Agent: %s\r\n", rule.user_agent), + "Connection: keep-alive\r\n", + "Encapsulated: null-body=0\r\n\r\n", + } + if rule.user_agent == "none" then + table.remove(options_request, 3) + end + + local respond_headers = { + -- Add main RESPMOD header before any other + string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), + } + + local size = tonumber(#content) + local chunked_size = string.format("%x", size) + + local function icap_callback(err, conn) + + local function icap_requery(err_m, info) + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + lua_util.debugm(rule.name, task, + '%s: %s Request Error: %s - retries left: %s', + rule.log_prefix, info, err_m, retransmits) + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) + + tcp_options.host = addr:to_string() + tcp_options.port = addr:get_port() + tcp_options.callback = icap_callback + tcp_options.data = options_request + tcp_options.upstream = upstream + + tcp.request(tcp_options) + + else + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - error: %s', rule.log_prefix, err_m or '') + common.yield_result(task, rule, string.format('failed - error: %s', err_m), + 0.0, 'fail', maybe_part) + end + end + + local function get_req_headers() + + local in_client_ip = task:get_from_ip() + local req_hlen = 2 + if maybe_part then + table.insert(req_headers, + string.format('GET http://%s/%s HTTP/1.0\r\n', in_client_ip, maybe_part:get_filename())) + if rule.use_specific_content_type then + table.insert(http_headers, string.format('Content-Type: %s/%s\r\n', maybe_part:get_detected_type())) + --else + -- To test: what content type is better for icap servers? + --table.insert(http_headers, 'Content-Type: text/plain\r\n') + end + else + table.insert(req_headers, string.format('GET %s HTTP/1.0\r\n', rule.req_fake_url)) + table.insert(http_headers, string.format('Content-Type: application/octet-stream\r\n')) + end + table.insert(req_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time()))) + if rule.user_agent ~= "none" then + table.insert(req_headers, string.format("User-Agent: %s\r\n", rule.user_agent)) + end + + for _, h in ipairs(req_headers) do + req_hlen = req_hlen + tonumber(#h) + end + + return req_hlen, req_headers + + end + + local function get_http_headers() + local http_hlen = 2 + table.insert(http_headers, 'HTTP/1.0 200 OK\r\n') + table.insert(http_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time()))) + table.insert(http_headers, string.format('Server: %s\r\n', 'Apache/2.4')) + if rule.user_agent ~= "none" then + table.insert(http_headers, string.format("User-Agent: %s\r\n", rule.user_agent)) + end + --table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html')) + table.insert(http_headers, string.format('Content-Length: %s\r\n', size)) + + for _, h in ipairs(http_headers) do + http_hlen = http_hlen + tonumber(#h) + end + + return http_hlen, http_headers + + end + + local function get_respond_query() + local req_hlen = 0 + local resp_req_headers + local http_hlen = 0 + local resp_http_headers + + -- Append all extra headers + if rule.user_agent ~= "none" then + table.insert(respond_headers, + string.format("User-Agent: %s\r\n", rule.user_agent)) + end + + if rule.req_headers_enabled then + req_hlen, resp_req_headers = get_req_headers() + end + if rule.http_headers_enabled then + http_hlen, resp_http_headers = get_http_headers() + end + + if rule.req_headers_enabled and rule.http_headers_enabled then + local res_body_hlen = req_hlen + http_hlen + table.insert(respond_headers, + string.format('Encapsulated: req-hdr=0, res-hdr=%s, res-body=%s\r\n', + req_hlen, res_body_hlen)) + elseif rule.http_headers_enabled then + table.insert(respond_headers, + string.format('Encapsulated: res-hdr=0, res-body=%s\r\n', + http_hlen)) + else + table.insert(respond_headers, 'Encapsulated: res-body=0\r\n') + end + + table.insert(respond_headers, '\r\n') + for _, h in ipairs(resp_req_headers) do + table.insert(respond_headers, h) + end + table.insert(respond_headers, '\r\n') + for _, h in ipairs(resp_http_headers) do + table.insert(respond_headers, h) + end + table.insert(respond_headers, '\r\n') + table.insert(respond_headers, chunked_size .. '\r\n') + table.insert(respond_headers, content) + table.insert(respond_headers, '\r\n0\r\n\r\n') + return respond_headers + end + + local function add_respond_header(name, value) + if name and value then + table.insert(respond_headers, string.format('%s: %s\r\n', name, value)) + end + end + + local function result_header_table(result) + local icap_headers = {} + for s in result:gmatch("[^\r\n]+") do + if string.find(s, '^ICAP') then + icap_headers['icap'] = tostring(s) + elseif string.find(s, '^HTTP') then + icap_headers['http'] = tostring(s) + elseif string.find(s, '[%a%d-+]-:') then + local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)") + if key ~= nil then + icap_headers[key:lower()] = tostring(value) + end + end + end + lua_util.debugm(rule.name, task, '%s: icap_headers: %s', + rule.log_prefix, icap_headers) + return icap_headers + end + + local function threat_table_add(icap_threat, maybe_split) + + if maybe_split and string.find(icap_threat, ',') then + local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {} + + for _, v in ipairs(threats) do + table.insert(threat_table, v) + end + else + table.insert(threat_table, icap_threat) + end + return true + end + + local function icap_parse_result(headers) + + --[[ + @ToDo: handle type in response + + Generic Strings: + icap: X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC; + icap: X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader; + + Symantec String: + icap: X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation + icap: X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation; + + Sophos Strings: + icap: X-Virus-ID: Troj/DocDl-OYC + http: X-Blocked: Virus found during virus scan + http: X-Blocked-By: Sophos Anti-Virus + + Kaspersky Web Traffic Security Strings: + icap: X-Virus-ID: HEUR:Backdoor.Java.QRat.gen + icap: X-Response-Info: blocked + icap: X-Virus-ID: no threats + icap: X-Response-Info: blocked + icap: X-Response-Info: passed + http: HTTP/1.1 403 Forbidden + + Kaspersky Scan Engine 2.0 (ICAP mode) + icap: X-Virus-ID: EICAR-Test-File + http: HTTP/1.0 403 Forbidden + + Trend Micro Strings: + icap: X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1 + icap: X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1; + http: HTTP/1.1 403 Forbidden (TMWS Blocked) + http: HTTP/1.1 403 Forbidden + + F-Secure Internet Gatekeeper Strings: + icap: X-FSecure-Scan-Result: infected + icap: X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203" + icap: X-FSecure-Infected-Filename: "virus.doc" + + ESET File Security for Linux 7.0 + icap: X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA; + icap: X-Virus-ID: Trojaner + icap: X-Response-Info: Blocked + + McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules) + icap: X-Virus-ID: EICAR test file + icap: X-Media-Type: text/plain + icap: X-Block-Result: 80 + icap: X-Block-Reason: Malware found + icap: X-Block-Reason: Archive not supported + icap: X-Block-Reason: Media Type (Block List) + http: HTTP/1.0 403 VirusFound + + C-ICAP Squidclamav + icap/http: X-Infection-Found: Type=0; Resolution=2; Threat={HEX}EICAR.TEST.3.UNOFFICIAL; + icap/http: X-Virus-ID: {HEX}EICAR.TEST.3.UNOFFICIAL + http: HTTP/1.0 307 Temporary Redirect + ]] -- + + -- Generic ICAP Headers + if headers['x-infection-found'] then + local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") + + -- Type=2 is typical for scan error returns + if icap_type and icap_type == '2' then + lua_util.debugm(rule.name, task, + '%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat) + common.yield_result(task, rule, icap_threat, 0, + 'fail', maybe_part) + return true + elseif icap_threat ~= nil then + lua_util.debugm(rule.name, task, + '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat) + threat_table_add(icap_threat, false) + -- stupid workaround for unuseable x-infection-found header + -- but also x-virus-name set (McAfee Web Gateway 9) + elseif not icap_threat and headers['x-virus-name'] then + threat_table_add(headers['x-virus-name'], true) + else + threat_table_add(headers['x-infection-found'], true) + end + elseif headers['x-virus-name'] and headers['x-virus-name'] ~= "no threats" then + lua_util.debugm(rule.name, task, + '%s: icap X-Virus-Name: %s', rule.log_prefix, headers['x-virus-name']) + threat_table_add(headers['x-virus-name'], true) + elseif headers['x-virus-id'] and headers['x-virus-id'] ~= "no threats" then + lua_util.debugm(rule.name, task, + '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id']) + threat_table_add(headers['x-virus-id'], true) + -- FSecure X-Headers + elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then + + local infected_filename = "" + local infection_name = "-unknown-" + + if headers['x-fsecure-infected-filename'] then + infected_filename = string.gsub(headers['x-fsecure-infected-filename'], '[%s"]', '') + end + if headers['x-fsecure-infection-name'] then + infection_name = string.gsub(headers['x-fsecure-infection-name'], '[%s"]', '') + end + + lua_util.debugm(rule.name, task, + '%s: icap X-FSecure-Infection-Name (X-FSecure-Infected-Filename): %s (%s)', + rule.log_prefix, infection_name, infected_filename) + + threat_table_add(infection_name, true) + -- McAfee Web Gateway manual extra headers + elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then + threat_table_add(headers['x-mwg-block-reason'], false) + -- Sophos SAVDI special http headers + elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then + threat_table_add(headers['x-blocked'], false) + elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then + threat_table_add(headers['x-block-reason'], false) + -- last try HTTP [4]xx return + elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then + threat_table_add( + string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + elseif rule.use_http_3xx_as_threat and + headers.http and + string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d') + then + threat_table_add( + string.format("pseudo-virus (redirect): %s", + string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + end + + if #threat_table > 0 then + common.yield_result(task, rule, threat_table, rule.default_score, nil, maybe_part) + common.save_cache(task, digest, rule, threat_table, rule.default_score, maybe_part) + return true + else + return false + end + end + + local function icap_r_respond_http_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_respond_http_cb") + else + local result = tostring(data) + + local icap_http_headers = result_header_table(result) or {} + -- Find HTTP/[12].x [234]xx response + if icap_http_headers.http and string.find(icap_http_headers.http, 'HTTP%/[12]%.. [234]%d%d') then + local icap_http_header_result = icap_parse_result(icap_http_headers) + if icap_http_header_result then + -- Threat found - close connection + connection:close() + else + common.save_cache(task, digest, rule, 'OK', 0, maybe_part) + common.log_clean(task, rule) + end + else + rspamd_logger.errx(task, '%s: unhandled response |%s|', + rule.log_prefix, string.gsub(result, "\r\n", ", ")) + common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap), + 0.0, 'fail', maybe_part) + end + end + end + + local function icap_r_respond_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_respond_cb") + else + local result = tostring(data) + + local icap_headers = result_header_table(result) or {} + -- Find ICAP/1.x 2xx response + if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then + local icap_header_result = icap_parse_result(icap_headers) + if icap_header_result then + -- Threat found - close connection + connection:close() + elseif not icap_header_result + and rule.use_http_result_header + and icap_headers.encapsulated + and not string.find(icap_headers.encapsulated, 'null%-body=0') + then + -- Try to read encapsulated HTTP Headers + lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers', + rule.log_prefix) + connection:add_read(icap_r_respond_http_cb, '\r\n\r\n') + else + connection:close() + common.save_cache(task, digest, rule, 'OK', 0, maybe_part) + common.log_clean(task, rule) + end + elseif icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then + -- Find ICAP/1.x 5/4xx response + --[[ + Symantec String: + ICAP/1.0 539 Aborted - No AV scanning license + SquidClamAV/C-ICAP: + ICAP/1.0 500 Server error + Eset: + ICAP/1.0 405 Forbidden + TrendMicro: + ICAP/1.0 400 Bad request + McAfee: + ICAP/1.0 418 Bad composition + ]]-- + rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap) + common.yield_result(task, rule, icap_headers.icap, 0.0, + 'fail', maybe_part) + return false + else + rspamd_logger.errx(task, '%s: unhandled response |%s|', + rule.log_prefix, string.gsub(result, "\r\n", ", ")) + common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap), + 0.0, 'fail', maybe_part) + end + end + end + + local function icap_w_respond_cb(err_m, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_w_respond_cb") + else + connection:add_read(icap_r_respond_cb, '\r\n\r\n') + end + end + + local function icap_r_options_cb(err_m, data, connection) + if err_m or connection == nil then + icap_requery(err_m, "icap_r_options_cb") + else + local icap_headers = result_header_table(tostring(data)) + + if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then + if icap_headers['methods'] and string.find(icap_headers['methods'], 'RESPMOD') then + -- Allow "204 No Content" responses + -- https://datatracker.ietf.org/doc/html/rfc3507#section-4.6 + if icap_headers['allow'] and string.find(icap_headers['allow'], '204') then + add_respond_header('Allow', '204') + end + + if rule.x_client_header then + local client = task:get_from_ip() + if client then + add_respond_header('X-Client-IP', client:to_string()) + end + end + + -- F-Secure extra headers + if icap_headers['server'] and string.find(icap_headers['server'], 'f-secure icap server') then + + if rule.x_rcpt_header then + local rcpt_to = task:get_principal_recipient() + if rcpt_to then + add_respond_header('X-Rcpt-To', rcpt_to) + end + end + + if rule.x_from_header then + local mail_from = task:get_principal_recipient() + if mail_from and mail_from[1] then + add_respond_header('X-Rcpt-To', mail_from[1].addr) + end + end + + end + + if icap_headers.connection and icap_headers.connection:lower() == 'close' then + lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection', + rule.log_prefix, icap_headers.connection) + connection:close() + tcp_options.callback = icap_w_respond_cb + tcp_options.data = get_respond_query() + tcp.request(tcp_options) + else + connection:add_write(icap_w_respond_cb, get_respond_query()) + end + + else + rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s', + rule.log_prefix, icap_headers['methods']) + common.yield_result(task, rule, 'NO RESPMOD', 0.0, + 'fail', maybe_part) + end + else + rspamd_logger.errx(task, '%s: OPTIONS query failed: %s', + rule.log_prefix, icap_headers.icap or "-") + common.yield_result(task, rule, 'OPTIONS query failed', 0.0, + 'fail', maybe_part) + end + end + end + + if err or conn == nil then + icap_requery(err, "options_request") + else + conn:add_read(icap_r_options_cb, '\r\n\r\n') + end + end + + tcp_options.task = task + tcp_options.stop_pattern = '\r\n' + tcp_options.read = false + tcp_options.timeout = rule.timeout + tcp_options.callback = icap_callback + tcp_options.data = options_request + + if rule.ssl then + tcp_options.ssl = true + if rule.no_ssl_verify then + tcp_options.no_ssl_verify = true + end + end + + tcp_options.host = addr:to_string() + tcp_options.port = addr:get_port() + tcp_options.upstream = upstream + + tcp.request(tcp_options) + end + + if common.condition_check_and_continue(task, content, rule, digest, + icap_check_uncached, maybe_part) then + return + else + icap_check_uncached() + end + +end + +return { + type = { N, 'virus', 'virus', 'scanner' }, + description = 'generic icap antivirus', + configure = icap_config, + check = icap_check, + name = N +} |