diff options
Diffstat (limited to 'rules/controller')
| -rw-r--r-- | rules/controller/fuzzy.lua | 46 | ||||
| -rw-r--r-- | rules/controller/init.lua | 67 | ||||
| -rw-r--r-- | rules/controller/maps.lua | 220 | ||||
| -rw-r--r-- | rules/controller/neural.lua | 70 | ||||
| -rw-r--r-- | rules/controller/selectors.lua | 73 |
5 files changed, 476 insertions, 0 deletions
diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua new file mode 100644 index 0000000..193e6fd --- /dev/null +++ b/rules/controller/fuzzy.lua @@ -0,0 +1,46 @@ +--[[ +Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local function handle_gen_fuzzy(task, conn, req_params) + if type(rspamd_plugins.fuzzy_check) == 'table' then + local ret, hashes + task:process_message() + if req_params.rule then + ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule) + elseif req_params.flag then + ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag)) + else + conn:send_error(404, 'missing rule or flag') + return + end + + if ret then + conn:send_ucl({ success = true, hashes = hashes }) + else + conn:send_error(500, 'cannot generate hashes') + end + else + conn:send_error(404, 'fuzzy_check is not enabled') + end +end + +return { + hashes = { + handler = handle_gen_fuzzy, + need_task = true, + enable = false + }, +}
\ No newline at end of file diff --git a/rules/controller/init.lua b/rules/controller/init.lua new file mode 100644 index 0000000..17fbbfc --- /dev/null +++ b/rules/controller/init.lua @@ -0,0 +1,67 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +-- Controller endpoints + +local local_conf = rspamd_paths['LOCAL_CONFDIR'] +local local_rules = rspamd_paths['RULESDIR'] +local rspamd_util = require "rspamd_util" +local lua_util = require "lua_util" +local rspamd_logger = require "rspamd_logger" + +-- Define default controller paths, could be overridden in local.d/controller.lua + +local controller_plugin_paths = { + maps = dofile(local_rules .. "/controller/maps.lua"), + neural = dofile(local_rules .. "/controller/neural.lua"), + selectors = dofile(local_rules .. "/controller/selectors.lua"), + fuzzy = dofile(local_rules .. "/controller/fuzzy.lua"), +} + +if rspamd_util.file_exists(local_conf .. '/controller.lua') then + local controller_overrides = dofile(local_conf .. '/controller.lua') + + if controller_overrides and type(controller_overrides) == 'table' then + controller_plugin_paths = lua_util.override_defaults(controller_plugin_paths, controller_overrides) + end +end + +for plug, paths in pairs(controller_plugin_paths) do + if not rspamd_plugins[plug] then + rspamd_plugins[plug] = {} + end + if not rspamd_plugins[plug].webui then + rspamd_plugins[plug].webui = {} + end + + local webui = rspamd_plugins[plug].webui + + for path, attrs in pairs(paths) do + if type(attrs) == 'table' then + if type(attrs.handler) ~= 'function' then + rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid handler: %s; ignore it', + plug, path, type(attrs.handler)) + else + webui[path] = lua_util.shallowcopy(attrs) + rspamd_logger.infox(rspamd_config, 'controller plugin %s; register webui path %s', + plug, path) + end + else + rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid type: %s; ignore it', + plug, path, type(attrs)) + end + end +end diff --git a/rules/controller/maps.lua b/rules/controller/maps.lua new file mode 100644 index 0000000..718e292 --- /dev/null +++ b/rules/controller/maps.lua @@ -0,0 +1,220 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +-- Controller maps plugin +local maps_cache +local maps_aliases +local lua_util = require "lua_util" +local ts = require("tableshape").types +local ucl = require "ucl" + +local function maybe_fill_maps_cache() + if not maps_cache then + maps_cache = {} + maps_aliases = {} + local maps = rspamd_config:get_maps() + for _, m in ipairs(maps) do + -- We get the first url here and that's it + local url = m:get_uri() + if url ~= 'static' then + if not maps_cache[url] then + local alias = url:match('/([^/]+)$') + maps_cache[url] = m + if not maps_aliases[alias] then + maps_aliases[alias] = url + end + else + -- Do not override, as we don't care about duplicate maps that come from different + -- sources. + -- In theory, that should be cached but there are some exceptions even so far... + url = math.random() -- to shut luacheck about empty branch with a comment + end + end + end + end +end + +local function check_specific_map(input, uri, m, results, report_misses) + local value = m:get_key(input) + + if value then + local result = { + map = uri, + alias = uri:match('/([^/]+)$'), + value = value, + key = input, + hit = true, + } + table.insert(results, result) + elseif report_misses then + local result = { + map = uri, + alias = uri:match('/([^/]+)$'), + key = input, + hit = false, + } + table.insert(results, result) + end +end + +local function handle_query_map(_, conn, req_params) + maybe_fill_maps_cache() + local keys_to_check = {} + + if req_params.value and req_params.value ~= '' then + keys_to_check[1] = req_params.value + elseif req_params.values then + keys_to_check = lua_util.str_split(req_params.values, ',') + end + + local results = {} + for _, key in ipairs(keys_to_check) do + for uri, m in pairs(maps_cache) do + check_specific_map(key, uri, m, results, req_params.report_misses) + end + end + conn:send_ucl { + success = (#results > 0), + results = results + } +end + +local function handle_query_specific_map(_, conn, req_params) + maybe_fill_maps_cache() + -- Fill keys to check + local keys_to_check = {} + if req_params.value and req_params.value ~= '' then + keys_to_check[1] = req_params.value + elseif req_params.values then + keys_to_check = lua_util.str_split(req_params.values, ',') + end + local maps_to_check = maps_cache + -- Fill maps to check + if req_params.maps then + local map_names = lua_util.str_split(req_params.maps, ',') + maps_to_check = {} + for _, mn in ipairs(map_names) do + if maps_cache[mn] then + maps_to_check[mn] = maps_cache[mn] + else + local alias = maps_aliases[mn] + + if alias then + maps_to_check[alias] = maps_cache[alias] + else + conn:send_error(404, 'no such map: ' .. mn) + end + end + end + end + + local results = {} + for _, key in ipairs(keys_to_check) do + for uri, m in pairs(maps_to_check) do + check_specific_map(key, uri, m, results, req_params.report_misses) + end + end + + conn:send_ucl { + success = (#results > 0), + results = results + } +end + +local function handle_list_maps(_, conn, _) + maybe_fill_maps_cache() + conn:send_ucl { + maps = lua_util.keys(maps_cache), + aliases = maps_aliases + } +end + +local query_json_schema = ts.shape { + maps = ts.array_of(ts.string):is_optional(), + report_misses = ts.boolean:is_optional(), + values = ts.array_of(ts.string), +} + +local function handle_query_json(task, conn) + maybe_fill_maps_cache() + + local parser = ucl.parser() + local ok, err = parser:parse_text(task:get_rawbody()) + if not ok then + conn:send_error(400, err) + return + end + local obj = parser:get_object() + + ok, err = query_json_schema:transform(obj) + if not ok then + conn:send_error(400, err) + return + end + + local maps_to_check = {} + local report_misses = obj.report_misses + local results = {} + + if obj.maps then + for _, mn in ipairs(obj.maps) do + if maps_cache[mn] then + maps_to_check[mn] = maps_cache[mn] + else + local alias = maps_aliases[mn] + + if alias then + maps_to_check[alias] = maps_cache[alias] + else + conn:send_error(400, 'no such map: ' .. mn) + return + end + end + end + else + maps_to_check = maps_cache + end + + for _, key in ipairs(obj.values) do + for uri, m in pairs(maps_to_check) do + check_specific_map(key, uri, m, results, report_misses) + end + end + conn:send_ucl { + success = (#results > 0), + results = results + } +end + +return { + query = { + handler = handle_query_map, + enable = false, + }, + query_json = { + handler = handle_query_json, + enable = false, + need_task = true, + }, + query_specific = { + handler = handle_query_specific_map, + enable = false, + }, + list = { + handler = handle_list_maps, + enable = false, + }, +} diff --git a/rules/controller/neural.lua b/rules/controller/neural.lua new file mode 100644 index 0000000..aef1042 --- /dev/null +++ b/rules/controller/neural.lua @@ -0,0 +1,70 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local neural_common = require "plugins/neural" +local ts = require("tableshape").types +local ucl = require "ucl" + +local E = {} + +-- Controller neural plugin + +local learn_request_schema = ts.shape { + ham_vec = ts.array_of(ts.array_of(ts.number)), + rule = ts.string:is_optional(), + spam_vec = ts.array_of(ts.array_of(ts.number)), +} + +local function handle_learn(task, conn) + local parser = ucl.parser() + local ok, err = parser:parse_text(task:get_rawbody()) + if not ok then + conn:send_error(400, err) + return + end + local req_params = parser:get_object() + + ok, err = learn_request_schema:transform(req_params) + if not ok then + conn:send_error(400, err) + return + end + + local rule_name = req_params.rule or 'default' + local rule = neural_common.settings.rules[rule_name] + local set = neural_common.get_rule_settings(task, rule) + local version = ((set.ann or E).version or 0) + 1 + + neural_common.spawn_train { + ev_base = task:get_ev_base(), + ann_key = neural_common.new_ann_key(rule, set, version), + set = set, + rule = rule, + ham_vec = req_params.ham_vec, + spam_vec = req_params.spam_vec, + worker = task:get_worker(), + } + + conn:send_string('{"success" : true}') +end + +return { + learn = { + handler = handle_learn, + enable = true, + need_task = true, + }, +} diff --git a/rules/controller/selectors.lua b/rules/controller/selectors.lua new file mode 100644 index 0000000..7fc2894 --- /dev/null +++ b/rules/controller/selectors.lua @@ -0,0 +1,73 @@ +--[[ +Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local lua_selectors = require "lua_selectors" + +-- Controller selectors plugin + +local function handle_list_transforms(_, conn) + conn:send_ucl(lua_selectors.list_transforms()) +end + +local function handle_list_extractors(_, conn) + conn:send_ucl(lua_selectors.list_extractors()) +end + +local function handle_check_selector(_, conn, req_params) + if req_params.selector and req_params.selector ~= '' then + local selector = lua_selectors.create_selector_closure(rspamd_config, + req_params.selector, '', true) + conn:send_ucl({ success = selector and true }) + else + conn:send_error(404, 'missing selector') + end +end + +local function handle_check_message(task, conn, req_params) + if req_params.selector and req_params.selector ~= '' then + local selector = lua_selectors.create_selector_closure(rspamd_config, + req_params.selector, '', true) + if not selector then + conn:send_error(500, 'invalid selector') + else + task:process_message() + local elts = selector(task) + conn:send_ucl({ success = true, data = elts }) + end + else + conn:send_error(404, 'missing selector') + end +end + +return { + list_extractors = { + handler = handle_list_extractors, + enable = true, + }, + list_transforms = { + handler = handle_list_transforms, + enable = true, + }, + check_selector = { + handler = handle_check_selector, + enable = true, + }, + check_message = { + handler = handle_check_message, + enable = true, + need_task = true, + } +} |