Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 476 insertions, 0 deletions

diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua
new file mode 100644
index 0000000..193e6fd
--- /dev/null
+++ b/rules/controller/fuzzy.lua
@@ -0,0 +1,46 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local function handle_gen_fuzzy(task, conn, req_params)
+  if type(rspamd_plugins.fuzzy_check) == 'table' then
+    local ret, hashes
+    task:process_message()
+    if req_params.rule then
+      ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule)
+    elseif req_params.flag then
+      ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag))
+    else
+      conn:send_error(404, 'missing rule or flag')
+      return
+    end
+
+    if ret then
+      conn:send_ucl({ success = true, hashes = hashes })
+    else
+      conn:send_error(500, 'cannot generate hashes')
+    end
+  else
+    conn:send_error(404, 'fuzzy_check is not enabled')
+  end
+end
+
+return {
+  hashes = {
+    handler = handle_gen_fuzzy,
+    need_task = true,
+    enable = false
+  },
+}
\ No newline at end of file
diff --git a/rules/controller/init.lua b/rules/controller/init.lua
new file mode 100644
index 0000000..17fbbfc
--- /dev/null
+++ b/rules/controller/init.lua
@@ -0,0 +1,67 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Controller endpoints
+
+local local_conf = rspamd_paths['LOCAL_CONFDIR']
+local local_rules = rspamd_paths['RULESDIR']
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+
+-- Define default controller paths, could be overridden in local.d/controller.lua
+
+local controller_plugin_paths = {
+  maps = dofile(local_rules .. "/controller/maps.lua"),
+  neural = dofile(local_rules .. "/controller/neural.lua"),
+  selectors = dofile(local_rules .. "/controller/selectors.lua"),
+  fuzzy = dofile(local_rules .. "/controller/fuzzy.lua"),
+}
+
+if rspamd_util.file_exists(local_conf .. '/controller.lua') then
+  local controller_overrides = dofile(local_conf .. '/controller.lua')
+
+  if controller_overrides and type(controller_overrides) == 'table' then
+    controller_plugin_paths = lua_util.override_defaults(controller_plugin_paths, controller_overrides)
+  end
+end
+
+for plug, paths in pairs(controller_plugin_paths) do
+  if not rspamd_plugins[plug] then
+    rspamd_plugins[plug] = {}
+  end
+  if not rspamd_plugins[plug].webui then
+    rspamd_plugins[plug].webui = {}
+  end
+
+  local webui = rspamd_plugins[plug].webui
+
+  for path, attrs in pairs(paths) do
+    if type(attrs) == 'table' then
+      if type(attrs.handler) ~= 'function' then
+        rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid handler: %s; ignore it',
+            plug, path, type(attrs.handler))
+      else
+        webui[path] = lua_util.shallowcopy(attrs)
+        rspamd_logger.infox(rspamd_config, 'controller plugin %s; register webui path %s',
+            plug, path)
+      end
+    else
+      rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid type: %s; ignore it',
+          plug, path, type(attrs))
+    end
+  end
+end
diff --git a/rules/controller/maps.lua b/rules/controller/maps.lua
new file mode 100644
index 0000000..718e292
--- /dev/null
+++ b/rules/controller/maps.lua
@@ -0,0 +1,220 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Controller maps plugin
+local maps_cache
+local maps_aliases
+local lua_util = require "lua_util"
+local ts = require("tableshape").types
+local ucl = require "ucl"
+
+local function maybe_fill_maps_cache()
+  if not maps_cache then
+    maps_cache = {}
+    maps_aliases = {}
+    local maps = rspamd_config:get_maps()
+    for _, m in ipairs(maps) do
+      -- We get the first url here and that's it
+      local url = m:get_uri()
+      if url ~= 'static' then
+        if not maps_cache[url] then
+          local alias = url:match('/([^/]+)$')
+          maps_cache[url] = m
+          if not maps_aliases[alias] then
+            maps_aliases[alias] = url
+          end
+        else
+          -- Do not override, as we don't care about duplicate maps that come from different
+          -- sources.
+          -- In theory, that should be cached but there are some exceptions even so far...
+          url = math.random() -- to shut luacheck about empty branch with a comment
+        end
+      end
+    end
+  end
+end
+
+local function check_specific_map(input, uri, m, results, report_misses)
+  local value = m:get_key(input)
+
+  if value then
+    local result = {
+      map = uri,
+      alias = uri:match('/([^/]+)$'),
+      value = value,
+      key = input,
+      hit = true,
+    }
+    table.insert(results, result)
+  elseif report_misses then
+    local result = {
+      map = uri,
+      alias = uri:match('/([^/]+)$'),
+      key = input,
+      hit = false,
+    }
+    table.insert(results, result)
+  end
+end
+
+local function handle_query_map(_, conn, req_params)
+  maybe_fill_maps_cache()
+  local keys_to_check = {}
+
+  if req_params.value and req_params.value ~= '' then
+    keys_to_check[1] = req_params.value
+  elseif req_params.values then
+    keys_to_check = lua_util.str_split(req_params.values, ',')
+  end
+
+  local results = {}
+  for _, key in ipairs(keys_to_check) do
+    for uri, m in pairs(maps_cache) do
+      check_specific_map(key, uri, m, results, req_params.report_misses)
+    end
+  end
+  conn:send_ucl {
+    success = (#results > 0),
+    results = results
+  }
+end
+
+local function handle_query_specific_map(_, conn, req_params)
+  maybe_fill_maps_cache()
+  -- Fill keys to check
+  local keys_to_check = {}
+  if req_params.value and req_params.value ~= '' then
+    keys_to_check[1] = req_params.value
+  elseif req_params.values then
+    keys_to_check = lua_util.str_split(req_params.values, ',')
+  end
+  local maps_to_check = maps_cache
+  -- Fill maps to check
+  if req_params.maps then
+    local map_names = lua_util.str_split(req_params.maps, ',')
+    maps_to_check = {}
+    for _, mn in ipairs(map_names) do
+      if maps_cache[mn] then
+        maps_to_check[mn] = maps_cache[mn]
+      else
+        local alias = maps_aliases[mn]
+
+        if alias then
+          maps_to_check[alias] = maps_cache[alias]
+        else
+          conn:send_error(404, 'no such map: ' .. mn)
+        end
+      end
+    end
+  end
+
+  local results = {}
+  for _, key in ipairs(keys_to_check) do
+    for uri, m in pairs(maps_to_check) do
+      check_specific_map(key, uri, m, results, req_params.report_misses)
+    end
+  end
+
+  conn:send_ucl {
+    success = (#results > 0),
+    results = results
+  }
+end
+
+local function handle_list_maps(_, conn, _)
+  maybe_fill_maps_cache()
+  conn:send_ucl {
+    maps = lua_util.keys(maps_cache),
+    aliases = maps_aliases
+  }
+end
+
+local query_json_schema = ts.shape {
+  maps = ts.array_of(ts.string):is_optional(),
+  report_misses = ts.boolean:is_optional(),
+  values = ts.array_of(ts.string),
+}
+
+local function handle_query_json(task, conn)
+  maybe_fill_maps_cache()
+
+  local parser = ucl.parser()
+  local ok, err = parser:parse_text(task:get_rawbody())
+  if not ok then
+    conn:send_error(400, err)
+    return
+  end
+  local obj = parser:get_object()
+
+  ok, err = query_json_schema:transform(obj)
+  if not ok then
+    conn:send_error(400, err)
+    return
+  end
+
+  local maps_to_check = {}
+  local report_misses = obj.report_misses
+  local results = {}
+
+  if obj.maps then
+    for _, mn in ipairs(obj.maps) do
+      if maps_cache[mn] then
+        maps_to_check[mn] = maps_cache[mn]
+      else
+        local alias = maps_aliases[mn]
+
+        if alias then
+          maps_to_check[alias] = maps_cache[alias]
+        else
+          conn:send_error(400, 'no such map: ' .. mn)
+          return
+        end
+      end
+    end
+  else
+    maps_to_check = maps_cache
+  end
+
+  for _, key in ipairs(obj.values) do
+    for uri, m in pairs(maps_to_check) do
+      check_specific_map(key, uri, m, results, report_misses)
+    end
+  end
+  conn:send_ucl {
+    success = (#results > 0),
+    results = results
+  }
+end
+
+return {
+  query = {
+    handler = handle_query_map,
+    enable = false,
+  },
+  query_json = {
+    handler = handle_query_json,
+    enable = false,
+    need_task = true,
+  },
+  query_specific = {
+    handler = handle_query_specific_map,
+    enable = false,
+  },
+  list = {
+    handler = handle_list_maps,
+    enable = false,
+  },
+}
diff --git a/rules/controller/neural.lua b/rules/controller/neural.lua
new file mode 100644
index 0000000..aef1042
--- /dev/null
+++ b/rules/controller/neural.lua
@@ -0,0 +1,70 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local neural_common = require "plugins/neural"
+local ts = require("tableshape").types
+local ucl = require "ucl"
+
+local E = {}
+
+-- Controller neural plugin
+
+local learn_request_schema = ts.shape {
+  ham_vec = ts.array_of(ts.array_of(ts.number)),
+  rule = ts.string:is_optional(),
+  spam_vec = ts.array_of(ts.array_of(ts.number)),
+}
+
+local function handle_learn(task, conn)
+  local parser = ucl.parser()
+  local ok, err = parser:parse_text(task:get_rawbody())
+  if not ok then
+    conn:send_error(400, err)
+    return
+  end
+  local req_params = parser:get_object()
+
+  ok, err = learn_request_schema:transform(req_params)
+  if not ok then
+    conn:send_error(400, err)
+    return
+  end
+
+  local rule_name = req_params.rule or 'default'
+  local rule = neural_common.settings.rules[rule_name]
+  local set = neural_common.get_rule_settings(task, rule)
+  local version = ((set.ann or E).version or 0) + 1
+
+  neural_common.spawn_train {
+    ev_base = task:get_ev_base(),
+    ann_key = neural_common.new_ann_key(rule, set, version),
+    set = set,
+    rule = rule,
+    ham_vec = req_params.ham_vec,
+    spam_vec = req_params.spam_vec,
+    worker = task:get_worker(),
+  }
+
+  conn:send_string('{"success" : true}')
+end
+
+return {
+  learn = {
+    handler = handle_learn,
+    enable = true,
+    need_task = true,
+  },
+}
diff --git a/rules/controller/selectors.lua b/rules/controller/selectors.lua
new file mode 100644
index 0000000..7fc2894
--- /dev/null
+++ b/rules/controller/selectors.lua
@@ -0,0 +1,73 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local lua_selectors = require "lua_selectors"
+
+-- Controller selectors plugin
+
+local function handle_list_transforms(_, conn)
+  conn:send_ucl(lua_selectors.list_transforms())
+end
+
+local function handle_list_extractors(_, conn)
+  conn:send_ucl(lua_selectors.list_extractors())
+end
+
+local function handle_check_selector(_, conn, req_params)
+  if req_params.selector and req_params.selector ~= '' then
+    local selector = lua_selectors.create_selector_closure(rspamd_config,
+        req_params.selector, '', true)
+    conn:send_ucl({ success = selector and true })
+  else
+    conn:send_error(404, 'missing selector')
+  end
+end
+
+local function handle_check_message(task, conn, req_params)
+  if req_params.selector and req_params.selector ~= '' then
+    local selector = lua_selectors.create_selector_closure(rspamd_config,
+        req_params.selector, '', true)
+    if not selector then
+      conn:send_error(500, 'invalid selector')
+    else
+      task:process_message()
+      local elts = selector(task)
+      conn:send_ucl({ success = true, data = elts })
+    end
+  else
+    conn:send_error(404, 'missing selector')
+  end
+end
+
+return {
+  list_extractors = {
+    handler = handle_list_extractors,
+    enable = true,
+  },
+  list_transforms = {
+    handler = handle_list_transforms,
+    enable = true,
+  },
+  check_selector = {
+    handler = handle_check_selector,
+    enable = true,
+  },
+  check_message = {
+    handler = handle_check_message,
+    enable = true,
+    need_task = true,
+  }
+}