Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1403 insertions, 0 deletions

diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
new file mode 100644
index 0000000..53b2732
--- /dev/null
+++ b/src/plugins/lua/multimap.lua
@@ -0,0 +1,1403 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+  return
+end
+
+-- Multimap is rspamd module designed to define and operate with different maps
+
+local rules = {}
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_regexp = require "rspamd_regexp"
+local rspamd_expression = require "rspamd_expression"
+local rspamd_ip = require "rspamd_ip"
+local lua_util = require "lua_util"
+local lua_selectors = require "lua_selectors"
+local lua_maps = require "lua_maps"
+local redis_params
+local fun = require "fun"
+local N = 'multimap'
+
+local multimap_grammar
+-- Parse result in form: <symbol>:<score>|<symbol>|<score>
+local function parse_multimap_value(parse_rule, p_ret)
+  if p_ret and type(p_ret) == 'string' then
+    local lpeg = require "lpeg"
+
+    if not multimap_grammar then
+      local number = {}
+
+      local digit = lpeg.R("09")
+      number.integer = (lpeg.S("+-") ^ -1) *
+          (digit ^ 1)
+
+      -- Matches: .6, .899, .9999873
+      number.fractional = (lpeg.P(".")) *
+          (digit ^ 1)
+
+      -- Matches: 55.97, -90.8, .9
+      number.decimal = (number.integer * -- Integer
+          (number.fractional ^ -1)) + -- Fractional
+          (lpeg.S("+-") * number.fractional)  -- Completely fractional number
+
+      local sym_start = lpeg.R("az", "AZ") + lpeg.S("_")
+      local sym_elt = sym_start + lpeg.R("09")
+      local symbol = sym_start * sym_elt ^ 0
+      local symbol_cap = lpeg.Cg(symbol, 'symbol')
+      local score_cap = lpeg.Cg(number.decimal, 'score')
+      local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol)) ^ 0), 'opts')
+      local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap)
+      local symscoreopt_cap = symscore_cap * lpeg.P(":") * opts_cap
+      local grammar = symscoreopt_cap + symscore_cap + symbol_cap + score_cap
+      multimap_grammar = lpeg.Ct(grammar)
+    end
+    local tbl = multimap_grammar:match(p_ret)
+
+    if tbl then
+      local sym
+      local score = 1.0
+      local opts = {}
+
+      if tbl.symbol then
+        sym = tbl.symbol
+      end
+      if tbl.score then
+        score = tonumber(tbl.score)
+      end
+      if tbl.opts then
+        opts = tbl.opts
+      end
+
+      return true, sym, score, opts
+    else
+      if p_ret ~= '' then
+        rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"',
+            parse_rule.symbol, p_ret)
+      end
+
+      return true, nil, 1.0, {}
+    end
+  elseif type(p_ret) == 'boolean' then
+    return p_ret, nil, 1.0, {}
+  end
+
+  return false, nil, 0.0, {}
+end
+
+local value_types = {
+  ip = {
+    get_value = function(ip)
+      return ip:to_string()
+    end,
+  },
+  from = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  helo = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  header = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  rcpt = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  user = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  url = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  dnsbl = {
+    get_value = function(ip)
+      return ip:to_string()
+    end,
+  },
+  filename = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  content = {
+    get_value = function()
+      return nil
+    end,
+  },
+  hostname = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  asn = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  country = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  received = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  mempool = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  selector = {
+    get_value = function(val)
+      return val
+    end,
+  },
+  symbol_options = {
+    get_value = function(val)
+      return val
+    end,
+  },
+}
+
+local function ip_to_rbl(ip, rbl)
+  return table.concat(ip:inversed_str_octets(), ".") .. '.' .. rbl
+end
+
+local function apply_hostname_filter(task, filter, hostname, r)
+  if filter == 'tld' then
+    local tld = rspamd_util.get_tld(hostname)
+    return tld
+  elseif filter == 'top' then
+    local tld = rspamd_util.get_tld(hostname)
+    return tld:match('[^.]*$') or tld
+  else
+    if not r['re_filter'] then
+      local pat = string.match(filter, 'tld:regexp:(.+)')
+      if not pat then
+        rspamd_logger.errx(task, 'bad search filter: %s', filter)
+        return
+      end
+      r['re_filter'] = rspamd_regexp.create_cached(pat)
+      if not r['re_filter'] then
+        rspamd_logger.errx(task, 'couldnt create regex: %s', pat)
+        return
+      end
+    end
+    local tld = rspamd_util.get_tld(hostname)
+    local res = r['re_filter']:search(tld)
+    if res then
+      return res[1]
+    else
+      return nil
+    end
+  end
+end
+
+local function apply_url_filter(task, filter, url, r)
+  if not filter then
+    return url:get_host()
+  end
+
+  if filter == 'tld' then
+    return url:get_tld()
+  elseif filter == 'top' then
+    local tld = url:get_tld()
+    return tld:match('[^.]*$') or tld
+  elseif filter == 'full' then
+    return url:get_text()
+  elseif filter == 'is_phished' then
+    if url:is_phished() then
+      return url:get_host()
+    else
+      return nil
+    end
+  elseif filter == 'is_redirected' then
+    if url:is_redirected() then
+      return url:get_host()
+    else
+      return nil
+    end
+  elseif filter == 'is_obscured' then
+    if url:is_obscured() then
+      return url:get_host()
+    else
+      return nil
+    end
+  elseif filter == 'path' then
+    return url:get_path()
+  elseif filter == 'query' then
+    return url:get_query()
+  elseif string.find(filter, 'tag:') then
+    local tags = url:get_tags()
+    local want_tag = string.match(filter, 'tag:(.*)')
+    for _, t in ipairs(tags) do
+      if t == want_tag then
+        return url:get_host()
+      end
+    end
+    return nil
+  elseif string.find(filter, 'tld:regexp:') then
+    if not r['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        r['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not r['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = r['re_filter']:search(url:get_tld())
+      if results then
+        return results[1]
+      else
+        return nil
+      end
+    end
+  elseif string.find(filter, 'full:regexp:') then
+    if not r['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        r['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not r['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = r['re_filter']:search(url:get_text())
+      if results then
+        return results[1]
+      else
+        return nil
+      end
+    end
+  elseif string.find(filter, 'regexp:') then
+    if not r['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        r['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not r['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = r['re_filter']:search(url:get_host())
+      if results then
+        return results[1]
+      else
+        return nil
+      end
+    end
+  elseif string.find(filter, '^template:') then
+    if not r['template'] then
+      r['template'] = string.match(filter, '^template:(.+)')
+    end
+
+    if r['template'] then
+      return lua_util.template(r['template'], url:to_table())
+    end
+  end
+
+  return url:get_host()
+end
+
+local function apply_addr_filter(task, filter, input, rule)
+  if filter == 'email:addr' or filter == 'email' then
+    local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+    if addr and addr[1] then
+      return fun.totable(fun.map(function(a)
+        return a.addr
+      end, addr))
+    end
+  elseif filter == 'email:user' then
+    local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+    if addr and addr[1] then
+      return fun.totable(fun.map(function(a)
+        return a.user
+      end, addr))
+    end
+  elseif filter == 'email:domain' then
+    local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+    if addr and addr[1] then
+      return fun.totable(fun.map(function(a)
+        return a.domain
+      end, addr))
+    end
+  elseif filter == 'email:domain:tld' then
+    local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+    if addr and addr[1] then
+      return fun.totable(fun.map(function(a)
+        return rspamd_util.get_tld(a.domain)
+      end, addr))
+    end
+  elseif filter == 'email:name' then
+    local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+    if addr and addr[1] then
+      return fun.totable(fun.map(function(a)
+        return a.name
+      end, addr))
+    end
+  elseif filter == 'ip_addr' then
+    local ip_addr = rspamd_ip.from_string(input)
+
+    if ip_addr and ip_addr:is_valid() then
+      return ip_addr
+    end
+  else
+    -- regexp case
+    if not rule['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        rule['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not rule['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = rule['re_filter']:search(input)
+      if results then
+        return results[1]
+      end
+    end
+  end
+
+  return input
+end
+local function apply_filename_filter(task, filter, fn, r)
+  if filter == 'extension' or filter == 'ext' then
+    return string.match(fn, '%.([^.]+)$')
+  elseif string.find(filter, 'regexp:') then
+    if not r['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        r['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not r['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = r['re_filter']:search(fn)
+      if results then
+        return results[1]
+      else
+        return nil
+      end
+    end
+  end
+
+  return fn
+end
+
+local function apply_regexp_filter(task, filter, fn, r)
+  if string.find(filter, 'regexp:') then
+    if not r['re_filter'] then
+      local type, pat = string.match(filter, '(regexp:)(.+)')
+      if type and pat then
+        r['re_filter'] = rspamd_regexp.create_cached(pat)
+      end
+    end
+
+    if not r['re_filter'] then
+      rspamd_logger.errx(task, 'bad search filter: %s', filter)
+    else
+      local results = r['re_filter']:search(fn, false, true)
+      if results then
+        return results[1][2]
+      else
+        return nil
+      end
+    end
+  end
+
+  return fn
+end
+
+local function apply_content_filter(task, filter)
+  if filter == 'body' then
+    return { task:get_rawbody() }
+  elseif filter == 'full' then
+    return { task:get_content() }
+  elseif filter == 'headers' then
+    return { task:get_raw_headers() }
+  elseif filter == 'text' then
+    local ret = {}
+    for _, p in ipairs(task:get_text_parts()) do
+      table.insert(ret, p:get_content())
+    end
+    return ret
+  elseif filter == 'rawtext' then
+    local ret = {}
+    for _, p in ipairs(task:get_text_parts()) do
+      table.insert(ret, p:get_content('raw_parsed'))
+    end
+    return ret
+  elseif filter == 'oneline' then
+    local ret = {}
+    for _, p in ipairs(task:get_text_parts()) do
+      table.insert(ret, p:get_content_oneline())
+    end
+    return ret
+  else
+    rspamd_logger.errx(task, 'bad search filter: %s', filter)
+  end
+
+  return {}
+end
+
+local multimap_filters = {
+  from = apply_addr_filter,
+  rcpt = apply_addr_filter,
+  helo = apply_hostname_filter,
+  symbol_options = apply_regexp_filter,
+  header = apply_addr_filter,
+  url = apply_url_filter,
+  filename = apply_filename_filter,
+  mempool = apply_regexp_filter,
+  selector = apply_regexp_filter,
+  hostname = apply_hostname_filter,
+  --content = apply_content_filter, -- Content filters are special :(
+}
+
+local function multimap_query_redis(key, task, value, callback)
+  local cmd = 'HGET'
+  if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
+    cmd = 'HMGET'
+  end
+
+  local srch = { key }
+
+  -- Insert all ips for some mask :(
+  if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
+    srch[#srch + 1] = tostring(value)
+    -- IPv6 case
+    local maxbits = 128
+    local minbits = 64
+    if value:get_version() == 4 then
+      maxbits = 32
+      minbits = 8
+    end
+    for i = maxbits, minbits, -1 do
+      local nip = value:apply_mask(i):tostring() .. "/" .. i
+      srch[#srch + 1] = nip
+    end
+  else
+    srch[#srch + 1] = value
+  end
+
+  local function redis_map_cb(err, data)
+    lua_util.debugm(N, task, 'got reply from Redis when trying to get key %s: err=%s, data=%s',
+        key, err, data)
+    if not err and type(data) ~= 'userdata' then
+      callback(data)
+    end
+  end
+
+  return rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      key, -- hash key
+      false, -- is write
+      redis_map_cb, --callback
+      cmd, -- command
+      srch -- arguments
+  )
+end
+
+local function multimap_callback(task, rule)
+  local function match_element(r, value, callback)
+    if not value then
+      return false
+    end
+
+    local function get_key_callback(ret, err_or_data, err_code)
+      lua_util.debugm(N, task, 'got return "%s" (err code = %s) for multimap %s',
+          err_or_data,
+          err_code,
+          rule.symbol)
+
+      if ret then
+        if type(err_or_data) == 'table' then
+          for _, elt in ipairs(err_or_data) do
+            callback(elt)
+          end
+        else
+          callback(err_or_data)
+        end
+      elseif err_code ~= 404 then
+        rspamd_logger.infox(task, "map %s: get key returned error %s: %s",
+            rule.symbol, err_code, err_or_data)
+      end
+    end
+
+    lua_util.debugm(N, task, 'check value %s for multimap %s', value,
+        rule.symbol)
+
+    local ret = false
+
+    if r.redis_key then
+      -- Deal with hash name here: it can be either plain string or a selector
+      if type(r.redis_key) == 'string' then
+        ret = multimap_query_redis(r.redis_key, task, value, callback)
+      else
+        -- Here we have a selector
+        local results = r.redis_key(task)
+
+        -- Here we need to spill this function into multiple queries
+        if type(results) == 'table' then
+          for _, res in ipairs(results) do
+            ret = multimap_query_redis(res, task, value, callback)
+
+            if not ret then
+              break
+            end
+          end
+        else
+          ret = multimap_query_redis(results, task, value, callback)
+        end
+      end
+
+      return ret
+    elseif r.map_obj then
+      r.map_obj:get_key(value, get_key_callback, task)
+    end
+  end
+
+  local function insert_results(result, opt)
+    local _, symbol, score, opts = parse_multimap_value(rule, result)
+    local forced = false
+    if symbol then
+      if rule.symbols_set then
+        if not rule.symbols_set[symbol] then
+          rspamd_logger.infox(task, 'symbol %s is not registered for map %s, ' ..
+              'replace it with just %s',
+              symbol, rule.symbol, rule.symbol)
+          symbol = rule.symbol
+        end
+      elseif rule.disable_multisymbol then
+        symbol = rule.symbol
+        if type(opt) == 'table' then
+          table.insert(opt, result)
+        elseif type(opt) ~= nil then
+          opt = { opt, result }
+        else
+          opt = { result }
+        end
+      else
+        forced = not rule.dynamic_symbols
+      end
+    else
+      symbol = rule.symbol
+    end
+
+    if opts and #opts > 0 then
+      -- Options come from the map itself
+      task:insert_result(forced, symbol, score, opts)
+    else
+      if opt then
+        if type(opt) == 'table' then
+          task:insert_result(forced, symbol, score, fun.totable(fun.map(tostring, opt)))
+        else
+          task:insert_result(forced, symbol, score, tostring(opt))
+        end
+
+      else
+        task:insert_result(forced, symbol, score)
+      end
+    end
+
+    if rule.action then
+      local message = rule.message
+      if rule.message_func then
+        message = rule.message_func(task, rule.symbol, opt)
+      end
+      if message then
+        task:set_pre_result(rule.action, message, N)
+      else
+        task:set_pre_result(rule.action, 'Matched map: ' .. rule.symbol, N)
+      end
+    end
+  end
+
+  -- Match a single value for against a single rule
+  local function match_rule(r, value)
+    local function rule_callback(result)
+      if result then
+        if type(result) == 'table' then
+          for _, rs in ipairs(result) do
+            if type(rs) ~= 'userdata' then
+              rule_callback(rs)
+            end
+          end
+          return
+        end
+        local opt = value_types[r['type']].get_value(value)
+        insert_results(result, opt)
+      end
+    end
+
+    if r.filter or r.type == 'url' then
+      local fn = multimap_filters[r.type]
+
+      if fn then
+
+        local filtered_value = fn(task, r.filter, value, r)
+        lua_util.debugm(N, task, 'apply filter %s for rule %s: %s -> %s',
+            r.filter, r.symbol, value, filtered_value)
+        value = filtered_value
+      end
+    end
+
+    if type(value) == 'table' then
+      fun.each(function(elt)
+        match_element(r, elt, rule_callback)
+      end, value)
+    else
+      match_element(r, value, rule_callback)
+    end
+  end
+
+  -- Match list of values according to the field
+  local function match_list(r, ls, fields)
+    if ls then
+      if fields then
+        fun.each(function(e)
+          local match = e[fields[1]]
+          if match then
+            if fields[2] then
+              match = fields[2](match)
+            end
+            match_rule(r, match)
+          end
+        end, ls)
+      else
+        fun.each(function(e)
+          match_rule(r, e)
+        end, ls)
+      end
+    end
+  end
+
+  local function match_addr(r, addr)
+    match_list(r, addr, { 'addr' })
+
+    if not r.filter then
+      match_list(r, addr, { 'domain' })
+      match_list(r, addr, { 'user' })
+    end
+  end
+
+  local function match_url(r, url)
+    match_rule(r, url)
+  end
+
+  local function match_hostname(r, hostname)
+    match_rule(r, hostname)
+  end
+
+  local function match_filename(r, fn)
+    match_rule(r, fn)
+  end
+
+  local function match_received_header(r, pos, total, h)
+    local use_tld = false
+    local filter = r['filter'] or 'real_ip'
+    if filter:match('^tld:') then
+      filter = filter:sub(5)
+      use_tld = true
+    end
+    local v = h[filter]
+    if v then
+      local min_pos = tonumber(r['min_pos'])
+      local max_pos = tonumber(r['max_pos'])
+      if min_pos then
+        if min_pos < 0 then
+          if min_pos == -1 then
+            if (pos ~= total) then
+              return
+            end
+          else
+            if pos <= (total - (min_pos * -1)) then
+              return
+            end
+          end
+        elseif pos < min_pos then
+          return
+        end
+      end
+      if max_pos then
+        if max_pos < -1 then
+          if (total - (max_pos * -1)) >= pos then
+            return
+          end
+        elseif max_pos > 0 then
+          if pos > max_pos then
+            return
+          end
+        end
+      end
+      local match_flags = r['flags']
+      local nmatch_flags = r['nflags']
+      if match_flags or nmatch_flags then
+        local got_flags = h['flags']
+        if match_flags then
+          for _, flag in ipairs(match_flags) do
+            if not got_flags[flag] then
+              return
+            end
+          end
+        end
+        if nmatch_flags then
+          for _, flag in ipairs(nmatch_flags) do
+            if got_flags[flag] then
+              return
+            end
+          end
+        end
+      end
+      if filter == 'real_ip' or filter == 'from_ip' then
+        if type(v) == 'string' then
+          v = rspamd_ip.from_string(v)
+        end
+        if v and v:is_valid() then
+          match_rule(r, v)
+        end
+      else
+        if use_tld and type(v) == 'string' then
+          v = rspamd_util.get_tld(v)
+        end
+        match_rule(r, v)
+      end
+    end
+  end
+
+  local function match_content(r)
+    local data
+
+    if r['filter'] then
+      data = apply_content_filter(task, r['filter'], r)
+    else
+      data = { task:get_content() }
+    end
+
+    for _, v in ipairs(data) do
+      match_rule(r, v)
+    end
+  end
+
+  if rule.expression and not rule.combined then
+    local res, trace = rule['expression']:process_traced(task)
+
+    if not res or res == 0 then
+      lua_util.debugm(N, task, 'condition is false for %s',
+          rule.symbol)
+      return
+    else
+      lua_util.debugm(N, task, 'condition is true for %s: %s',
+          rule.symbol,
+          trace)
+    end
+  end
+
+  local process_rule_funcs = {
+    ip = function()
+      local ip = task:get_from_ip()
+      if ip and ip:is_valid() then
+        match_rule(rule, ip)
+      end
+    end,
+    dnsbl = function()
+      local ip = task:get_from_ip()
+      if ip and ip:is_valid() then
+        local to_resolve = ip_to_rbl(ip, rule['map'])
+        local function dns_cb(_, _, results, err)
+          lua_util.debugm(N, rspamd_config,
+              'resolve() finished: results=%1, err=%2, to_resolve=%3',
+              results, err, to_resolve)
+
+          if err and
+              (err ~= 'requested record is not found' and
+                  err ~= 'no records with this name') then
+            rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
+          elseif results then
+            task:insert_result(rule['symbol'], 1, rule['map'])
+            if rule.action then
+              task:set_pre_result(rule['action'],
+                  'Matched map: ' .. rule['symbol'], N)
+            end
+          end
+        end
+
+        task:get_resolver():resolve_a({
+          task = task,
+          name = to_resolve,
+          callback = dns_cb,
+          forced = true
+        })
+      end
+    end,
+    header = function()
+      if type(rule['header']) == 'table' then
+        for _, rh in ipairs(rule['header']) do
+          local hv = task:get_header_full(rh)
+          match_list(rule, hv, { 'decoded' })
+        end
+      else
+        local hv = task:get_header_full(rule['header'])
+        match_list(rule, hv, { 'decoded' })
+      end
+    end,
+    rcpt = function()
+      if task:has_recipients('smtp') then
+        local rcpts = task:get_recipients('smtp')
+        match_addr(rule, rcpts)
+      elseif task:has_recipients('mime') then
+        local rcpts = task:get_recipients('mime')
+        match_addr(rule, rcpts)
+      end
+    end,
+    from = function()
+      if task:has_from('smtp') then
+        local from = task:get_from('smtp')
+        match_addr(rule, from)
+      elseif task:has_from('mime') then
+        local from = task:get_from('mime')
+        match_addr(rule, from)
+      end
+    end,
+    helo = function()
+      local helo = task:get_helo()
+      if helo then
+        match_hostname(rule, helo)
+      end
+    end,
+    url = function()
+      if task:has_urls() then
+        local msg_urls = task:get_urls()
+
+        for _, url in ipairs(msg_urls) do
+          match_url(rule, url)
+        end
+      end
+    end,
+    user = function()
+      local user = task:get_user()
+      if user then
+        match_rule(rule, user)
+      end
+    end,
+    filename = function()
+      local parts = task:get_parts()
+
+      local function filter_parts(p)
+        return p:is_attachment() or (not p:is_text()) and (not p:is_multipart())
+      end
+
+      local function filter_archive(p)
+        local ext = p:get_detected_ext()
+        local det_type = 'unknown'
+
+        if ext then
+          local lua_magic_types = require "lua_magic/types"
+          local det_t = lua_magic_types[ext]
+
+          if det_t then
+            det_type = det_t.type
+          end
+        end
+
+        return p:is_archive() and det_type == 'archive' and not rule.skip_archives
+      end
+
+      for _, p in fun.iter(fun.filter(filter_parts, parts)) do
+        if filter_archive(p) then
+          local fnames = p:get_archive():get_files(1000)
+
+          for _, fn in ipairs(fnames) do
+            match_filename(rule, fn)
+          end
+        end
+
+        local fn = p:get_filename()
+        if fn then
+          match_filename(rule, fn)
+        end
+        -- Also deal with detected content type
+        if not rule.skip_detected then
+          local ext = p:get_detected_ext()
+
+          if ext then
+            local fake_fname = string.format('detected.%s', ext)
+            lua_util.debugm(N, task, 'detected filename %s',
+                fake_fname)
+            match_filename(rule, fake_fname)
+          end
+        end
+      end
+    end,
+
+    content = function()
+      match_content(rule)
+    end,
+    hostname = function()
+      local hostname = task:get_hostname()
+      if hostname then
+        match_hostname(rule, hostname)
+      end
+    end,
+    asn = function()
+      local asn = task:get_mempool():get_variable('asn')
+      if asn then
+        match_rule(rule, asn)
+      end
+    end,
+    country = function()
+      local country = task:get_mempool():get_variable('country')
+      if country then
+        match_rule(rule, country)
+      end
+    end,
+    mempool = function()
+      local var = task:get_mempool():get_variable(rule['variable'])
+      if var then
+        match_rule(rule, var)
+      end
+    end,
+    symbol_options = function()
+      local sym = task:get_symbol(rule['target_symbol'])
+      if sym and sym[1].options then
+        for _, o in ipairs(sym[1].options) do
+          match_rule(rule, o)
+        end
+      end
+    end,
+    received = function()
+      local hdrs = task:get_received_headers()
+      if hdrs and hdrs[1] then
+        if not rule['artificial'] then
+          hdrs = fun.filter(function(h)
+            return not h['flags']['artificial']
+          end, hdrs):totable()
+        end
+        for pos, h in ipairs(hdrs) do
+          match_received_header(rule, pos, #hdrs, h)
+        end
+      end
+    end,
+    selector = function()
+      local elts = rule.selector(task)
+
+      if elts then
+        if type(elts) == 'table' then
+          for _, elt in ipairs(elts) do
+            match_rule(rule, elt)
+          end
+        else
+          match_rule(rule, elts)
+        end
+      end
+    end,
+    combined = function()
+      local ret, trace = rule.combined:process(task)
+      if ret and ret ~= 0 then
+        for n, t in pairs(trace) do
+          insert_results(t.value, string.format("%s=%s",
+              n, t.matched))
+        end
+      end
+    end,
+  }
+
+  local rt = rule.type
+  local process_func = process_rule_funcs[rt]
+  if process_func then
+    process_func()
+  else
+    rspamd_logger.errx(task, 'Unrecognised rule type: %s', rt)
+  end
+end
+
+local function gen_multimap_callback(rule)
+  return function(task)
+    multimap_callback(task, rule)
+  end
+end
+
+local function multimap_on_load_gen(rule)
+  return function()
+    lua_util.debugm(N, rspamd_config, "loaded map object for rule %s", rule['symbol'])
+    local known_symbols = {}
+    rule.map_obj:foreach(function(key, value)
+      local r, symbol, score, _ = parse_multimap_value(rule, value)
+
+      if r and symbol and not known_symbols[symbol] then
+        lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s",
+            rule.symbol, symbol, score, key)
+        rspamd_config:register_symbol {
+          name = value,
+          parent = rule.callback_id,
+          type = 'virtual',
+          score = score,
+        }
+        rspamd_config:set_metric_symbol({
+          group = N,
+          score = 1.0, -- In future, we will parse score from `get_value` and use it as multiplier
+          description = 'Automatic symbol generated by rule: ' .. rule.symbol,
+          name = value,
+        })
+        known_symbols[value] = true
+      end
+    end)
+  end
+end
+
+local function add_multimap_rule(key, newrule)
+  local ret = false
+
+  local function multimap_load_kv_map(rule)
+    if rule['regexp'] then
+      if rule['multi'] then
+        rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp_multi',
+            rule.description)
+      else
+        rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp',
+            rule.description)
+      end
+    elseif rule['glob'] then
+      if rule['multi'] then
+        rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob_multi',
+            rule.description)
+      else
+        rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob',
+            rule.description)
+      end
+    else
+      rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'hash',
+          rule.description)
+    end
+  end
+
+  local known_generic_types = {
+    header = true,
+    rcpt = true,
+    from = true,
+    helo = true,
+    symbol_options = true,
+    filename = true,
+    url = true,
+    user = true,
+    content = true,
+    hostname = true,
+    asn = true,
+    country = true,
+    mempool = true,
+    selector = true,
+    combined = true
+  }
+
+  if newrule['message_func'] then
+    newrule['message_func'] = assert(load(newrule['message_func']))()
+  end
+  if newrule['url'] and not newrule['map'] then
+    newrule['map'] = newrule['url']
+  end
+  if not (newrule.map or newrule.rules) then
+    rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map')
+    return nil
+  end
+  if not newrule['symbol'] and key then
+    newrule['symbol'] = key
+  elseif not newrule['symbol'] then
+    rspamd_logger.errx(rspamd_config, 'incomplete rule, missing symbol')
+    return nil
+  end
+  if not newrule['description'] then
+    newrule['description'] = string.format('multimap, type %s: %s', newrule['type'],
+        newrule['symbol'])
+  end
+  if newrule['type'] == 'mempool' and not newrule['variable'] then
+    rspamd_logger.errx(rspamd_config, 'mempool map requires variable')
+    return nil
+  end
+  if newrule['type'] == 'selector' then
+    if not newrule['selector'] then
+      rspamd_logger.errx(rspamd_config, 'selector map requires selector definition')
+      return nil
+    else
+      local selector = lua_selectors.create_selector_closure(
+          rspamd_config, newrule['selector'], newrule['delimiter'] or "")
+
+      if not selector then
+        rspamd_logger.errx(rspamd_config, 'selector map has invalid selector: "%s", symbol: %s',
+            newrule['selector'], newrule['symbol'])
+        return nil
+      end
+
+      newrule.selector = selector
+    end
+  end
+  if type(newrule['map']) == 'string' and
+      string.find(newrule['map'], '^redis://.*$') then
+    if not redis_params then
+      rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
+          'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+      return nil
+    end
+
+    newrule['redis_key'] = string.match(newrule['map'], '^redis://(.*)$')
+
+    if newrule['redis_key'] then
+      ret = true
+    end
+  elseif type(newrule['map']) == 'string' and
+      string.find(newrule['map'], '^redis%+selector://.*$') then
+    if not redis_params then
+      rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
+          'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+      return nil
+    end
+
+    local selector_str = string.match(newrule['map'], '^redis%+selector://(.*)$')
+    local selector = lua_selectors.create_selector_closure(
+        rspamd_config, selector_str, newrule['delimiter'] or "")
+
+    if not selector then
+      rspamd_logger.errx(rspamd_config, 'redis selector map has invalid selector: "%s", symbol: %s',
+          selector_str, newrule['symbol'])
+      return nil
+    end
+
+    newrule['redis_key'] = selector
+    ret = true
+  elseif newrule.type == 'combined' then
+    local lua_maps_expressions = require "lua_maps_expressions"
+    newrule.combined = lua_maps_expressions.create(rspamd_config,
+        {
+          rules = newrule.rules,
+          expression = newrule.expression,
+          on_load = newrule.dynamic_symbols and multimap_on_load_gen(newrule) or nil,
+        }, N, 'Combined map for ' .. newrule.symbol)
+    if not newrule.combined then
+      rspamd_logger.errx(rspamd_config, 'cannot add combined map for %s', newrule.symbol)
+    else
+      ret = true
+    end
+  else
+    if newrule['type'] == 'ip' then
+      newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+          newrule.description)
+      if newrule.map_obj then
+        ret = true
+      else
+        rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+            newrule['map'])
+      end
+    elseif newrule['type'] == 'received' then
+      if type(newrule['flags']) == 'table' and newrule['flags'][1] then
+        newrule['flags'] = newrule['flags']
+      elseif type(newrule['flags']) == 'string' then
+        newrule['flags'] = { newrule['flags'] }
+      end
+      if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then
+        newrule['nflags'] = newrule['nflags']
+      elseif type(newrule['nflags']) == 'string' then
+        newrule['nflags'] = { newrule['nflags'] }
+      end
+      local filter = newrule['filter'] or 'real_ip'
+      if filter == 'real_ip' or filter == 'from_ip' then
+        newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+            newrule.description)
+        if newrule.map_obj then
+          ret = true
+        else
+          rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+              newrule['map'])
+        end
+      else
+        multimap_load_kv_map(newrule)
+
+        if newrule.map_obj then
+          ret = true
+        else
+          rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+              newrule['map'])
+        end
+      end
+    elseif known_generic_types[newrule.type] then
+
+      if newrule.filter == 'ip_addr' then
+        newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+            newrule.description)
+      elseif not newrule.combined then
+        multimap_load_kv_map(newrule)
+      end
+
+      if newrule.map_obj then
+        ret = true
+      else
+        rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+            newrule['map'])
+      end
+    elseif newrule['type'] == 'dnsbl' then
+      ret = true
+    end
+  end
+
+  if ret then
+    if newrule.map_obj and newrule.dynamic_symbols then
+      newrule.map_obj:on_load(multimap_on_load_gen(newrule))
+    end
+    if newrule['type'] == 'symbol_options' then
+      rspamd_config:register_dependency(newrule['symbol'], newrule['target_symbol'])
+    end
+    if newrule['require_symbols'] then
+      local atoms = {}
+
+      local function parse_atom(str)
+        local atom = table.concat(fun.totable(fun.take_while(function(c)
+          if string.find(', \t()><+!|&\n', c, 1, true) then
+            return false
+          end
+          return true
+        end, fun.iter(str))), '')
+        table.insert(atoms, atom)
+        return atom
+      end
+
+      local function process_atom(atom, task)
+        local f_ret = task:has_symbol(atom)
+        lua_util.debugm(N, rspamd_config, 'check for symbol %s: %s', atom, f_ret)
+
+        if f_ret then
+          return 1
+        end
+
+        return 0
+      end
+
+      local expression = rspamd_expression.create(newrule['require_symbols'],
+          { parse_atom, process_atom }, rspamd_config:get_mempool())
+      if expression then
+        newrule['expression'] = expression
+
+        fun.each(function(v)
+          lua_util.debugm(N, rspamd_config, 'add dependency %s -> %s',
+              newrule['symbol'], v)
+          rspamd_config:register_dependency(newrule['symbol'], v)
+        end, atoms)
+      end
+    end
+    return newrule
+  end
+
+  return nil
+end
+
+-- Registration
+local opts = rspamd_config:get_all_opt(N)
+if opts and type(opts) == 'table' then
+  redis_params = rspamd_parse_redis_server(N)
+  for k, m in pairs(opts) do
+    if type(m) == 'table' and m['type'] then
+      local rule = add_multimap_rule(k, m)
+      if not rule then
+        rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
+      else
+        rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)',
+            k, rule.type)
+        table.insert(rules, rule)
+      end
+    end
+  end
+  -- add fake symbol to check all maps inside a single callback
+  fun.each(function(rule)
+    local augmentations = {}
+
+    if rule.action then
+      table.insert(augmentations, 'passthrough')
+    end
+
+    local id = rspamd_config:register_symbol({
+      type = 'normal',
+      name = rule['symbol'],
+      augmentations = augmentations,
+      callback = gen_multimap_callback(rule),
+    })
+
+    rule.callback_id = id
+
+    if rule['symbols'] then
+      -- Find allowed symbols by this map
+      rule['symbols_set'] = {}
+      fun.each(function(s)
+        rspamd_config:register_symbol({
+          type = 'virtual',
+          name = s,
+          parent = id,
+          score = tonumber(rule.score or "0") or 0, -- Default score
+        })
+        rule['symbols_set'][s] = 1
+      end, rule['symbols'])
+    end
+    if not rule.score then
+      rspamd_logger.infox(rspamd_config, 'set default score 0 for multimap rule %s', rule.symbol)
+      rule.score = 0
+    end
+    if rule.score then
+      -- Register metric symbol
+      rule.name = rule.symbol
+      rule.description = rule.description or 'multimap symbol'
+      rule.group = rule.group or N
+
+      local tmp_flags
+      tmp_flags = rule.flags
+
+      if rule.type == 'received' and rule.flags then
+        -- XXX: hack to allow received flags/nflags
+        -- See issue #3526 on GH
+        rule.flags = nil
+      end
+
+      -- XXX: for combined maps we use trace, so flags must include one_shot to avoid scores multiplication
+      if rule.combined and not rule.flags then
+        rule.flags = 'one_shot'
+      end
+      rspamd_config:set_metric_symbol(rule)
+      rule.flags = tmp_flags
+    end
+  end, rules)
+
+  if #rules == 0 then
+    lua_util.disable_module(N, "config")
+  end
+end