Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 355 insertions, 0 deletions

diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua
new file mode 100644
index 0000000..986d1a0
--- /dev/null
+++ b/lualib/lua_fuzzy.lua
@@ -0,0 +1,355 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_fuzzy
+-- This module contains helper functions for supporting fuzzy check module
+--]]
+
+
+local N = "lua_fuzzy"
+local lua_util = require "lua_util"
+local rspamd_regexp = require "rspamd_regexp"
+local fun = require "fun"
+local rspamd_logger = require "rspamd_logger"
+local ts = require("tableshape").types
+
+-- Filled by C code, indexed by number in this table
+local rules = {}
+
+-- Pre-defined rules options
+local policies = {
+  recommended = {
+    min_bytes = 1024,
+    min_height = 500,
+    min_width = 500,
+    min_length = 64,
+    text_multiplier = 4.0, -- divide min_bytes by 4 for texts
+    mime_types = { "application/*" },
+    scan_archives = true,
+    short_text_direct_hash = true,
+    text_shingles = true,
+    skip_images = false,
+  }
+}
+
+local default_policy = policies.recommended
+
+local schema_fields = {
+  min_bytes = ts.number + ts.string / tonumber,
+  min_height = ts.number + ts.string / tonumber,
+  min_width = ts.number + ts.string / tonumber,
+  min_length = ts.number + ts.string / tonumber,
+  text_multiplier = ts.number,
+  mime_types = ts.array_of(ts.string),
+  scan_archives = ts.boolean,
+  short_text_direct_hash = ts.boolean,
+  text_shingles = ts.boolean,
+  skip_images = ts.boolean,
+}
+local policy_schema = ts.shape(schema_fields)
+
+local policy_schema_open = ts.shape(schema_fields, {
+  open = true,
+})
+
+local exports = {}
+
+
+--[[[
+-- @function lua_fuzzy.register_policy(name, policy)
+-- Adds a new policy with name `name`. Must be valid, checked using policy_schema
+--]]
+exports.register_policy = function(name, policy)
+  if policies[name] then
+    rspamd_logger.warnx(rspamd_config, "overriding policy %s", name)
+  end
+
+  local parsed_policy, err = policy_schema:transform(policy)
+
+  if not parsed_policy then
+    rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s',
+        name, err)
+
+    return
+  else
+    policies.name = parsed_policy
+  end
+end
+
+--[[[
+-- @function lua_fuzzy.process_rule(rule)
+-- Processes fuzzy rule (applying policies or defaults if needed). Returns policy id
+--]]
+exports.process_rule = function(rule)
+  local processed_rule = lua_util.shallowcopy(rule)
+  local policy = default_policy
+
+  if processed_rule.policy then
+    policy = policies[processed_rule.policy]
+  end
+
+  if policy then
+    processed_rule = lua_util.override_defaults(policy, processed_rule)
+
+    local parsed_policy, err = policy_schema_open:transform(processed_rule)
+
+    if not parsed_policy then
+      rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule default fields: %s', err)
+    else
+      processed_rule = parsed_policy
+    end
+  else
+    rspamd_logger.warnx(rspamd_config, "unknown policy %s", processed_rule.policy)
+  end
+
+  if processed_rule.mime_types then
+    processed_rule.mime_types = fun.totable(fun.map(function(gl)
+      return rspamd_regexp.import_glob(gl, 'i')
+    end, processed_rule.mime_types))
+  end
+
+  table.insert(rules, processed_rule)
+  return #rules
+end
+
+local function check_length(task, part, rule)
+  local bytes = part:get_length()
+  local length_ok = bytes > 0
+
+  local id = part:get_id()
+  lua_util.debugm(N, task, 'check size of part %s', id)
+
+  if length_ok and rule.min_bytes > 0 then
+
+    local adjusted_bytes = bytes
+
+    if part:is_text() then
+      -- Fuzzy plugin uses stripped utf content to get an exact hash, that
+      -- corresponds to `get_content_oneline()`
+      -- However, in the case of empty parts this method returns `nil`, so extra
+      -- sanity check is required.
+      bytes = #(part:get_text():get_content_oneline() or '')
+
+      -- Short hashing algorithm also use subject unless explicitly denied
+      if not rule.no_subject then
+        local subject = task:get_subject() or ''
+        bytes = bytes + #subject
+      end
+
+      if rule.text_multiplier then
+        adjusted_bytes = bytes * rule.text_multiplier
+      end
+    end
+
+    if rule.min_bytes > adjusted_bytes then
+      lua_util.debugm(N, task, 'skip part of length %s (%s adjusted) ' ..
+          'as it has less than %s bytes',
+          bytes, adjusted_bytes, rule.min_bytes)
+      length_ok = false
+    else
+      lua_util.debugm(N, task, 'allow part of length %s (%s adjusted)',
+          bytes, adjusted_bytes, rule.min_bytes)
+    end
+  else
+    lua_util.debugm(N, task, 'allow part %s, no length limits', id)
+  end
+
+  return length_ok
+end
+
+local function check_text_part(task, part, rule, text)
+  local allow_direct, allow_shingles = false, false
+
+  local id = part:get_id()
+  lua_util.debugm(N, task, 'check text part %s', id)
+  local wcnt = text:get_words_count()
+
+  if rule.text_shingles then
+    -- Check number of words
+    local min_words = rule.min_length or 0
+    if min_words < 32 then
+      min_words = 32 -- Minimum for shingles
+    end
+    if wcnt < min_words then
+      lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles',
+          rule.min_length, wcnt)
+      allow_shingles = false
+    else
+      lua_util.debugm(N, task, 'allow shingles in text %s, %s words',
+          id, wcnt)
+      allow_shingles = true
+    end
+
+    if not rule.short_text_direct_hash and not allow_shingles then
+      allow_direct = false
+    else
+      if not allow_shingles then
+        lua_util.debugm(N, task,
+            'allow direct hash for short text %s, %s words',
+            id, wcnt)
+        allow_direct = check_length(task, part, rule)
+      else
+        allow_direct = wcnt > 0
+      end
+    end
+  else
+    lua_util.debugm(N, task,
+        'disable shingles in text %s', id)
+    allow_direct = check_length(task, part, rule)
+  end
+
+  return allow_direct, allow_shingles
+end
+
+--local function has_sane_text_parts(task)
+--  local text_parts = task:get_text_parts() or {}
+--  return fun.any(function(tp) return tp:get_words_count() > 32 end, text_parts)
+--end
+
+local function check_image_part(task, part, rule, image)
+  if rule.skip_images then
+    lua_util.debugm(N, task, 'skip image part as images are disabled')
+    return false, false
+  end
+
+  local id = part:get_id()
+  lua_util.debugm(N, task, 'check image part %s', id)
+
+  if rule.min_width > 0 or rule.min_height > 0 then
+    -- Check dimensions
+    local min_width = rule.min_width or rule.min_height
+    local min_height = rule.min_height or rule.min_width
+    local height = image:get_height()
+    local width = image:get_width()
+
+    if height and width then
+      if height < min_height or width < min_width then
+        lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s',
+            id, width, height, min_width, min_height)
+        return false, false
+      else
+        lua_util.debugm(N, task, 'allow image part %s: %sx%s',
+            id, width, height)
+      end
+    end
+  end
+
+  return check_length(task, part, rule), false
+end
+
+local function mime_types_check(task, part, rule)
+  local t, st = part:get_type()
+
+  if not t then
+    return false, false
+  end
+
+  local ct = string.format('%s/%s', t, st)
+
+  local detected_ct
+  t, st = part:get_detected_type()
+  if t then
+    detected_ct = string.format('%s/%s', t, st)
+  else
+    detected_ct = ct
+  end
+
+  local id = part:get_id()
+  lua_util.debugm(N, task, 'check binary part %s: %s', id, ct)
+
+  -- For bad mime parts we implicitly enable fuzzy check
+  local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1]
+  local opts = {}
+
+  if mime_trace then
+    opts = mime_trace.options or opts
+  end
+  opts = fun.tomap(fun.map(function(opt)
+    local elts = lua_util.str_split(opt, ':')
+    return elts[1], elts[2]
+  end, opts))
+
+  if opts[id] and opts[id] == '-' then
+    lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct)
+    return check_length(task, part, rule), false
+  end
+
+  if rule.mime_types then
+
+    if fun.any(function(gl_re)
+      if gl_re:match(ct) or (detected_ct and gl_re:match(detected_ct)) then
+        return true
+      else
+        return false
+      end
+    end, rule.mime_types) then
+      lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)',
+          id, ct, detected_ct)
+      return check_length(task, part, rule), false
+    end
+
+    return false, false
+  end
+
+  return false, false
+end
+
+exports.check_mime_part = function(task, part, rule_id)
+  local rule = rules[rule_id]
+
+  if not rule then
+    rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id)
+
+    return false, false
+  end
+
+  if part:is_text() then
+    return check_text_part(task, part, rule, part:get_text())
+  end
+
+  if part:is_image() then
+    return check_image_part(task, part, rule, part:get_image())
+  end
+
+  if part:is_archive() and rule.scan_archives then
+    -- Always send archives
+    lua_util.debugm(N, task, 'check archive part %s', part:get_id())
+
+    return true, false
+  end
+
+  if part:is_specific() then
+    local sp = part:get_specific()
+
+    if type(sp) == 'table' and sp.fuzzy_hashes then
+      lua_util.debugm(N, task, 'check specific part %s', part:get_id())
+      return true, false
+    end
+  end
+
+  if part:is_attachment() then
+    return mime_types_check(task, part, rule)
+  end
+
+  return false, false
+end
+
+exports.cleanup_rules = function()
+  rules = {}
+end
+
+return exports