Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1425 insertions, 0 deletions

diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
new file mode 100644
index 0000000..b2ccf86
--- /dev/null
+++ b/src/plugins/lua/rbl.lua
@@ -0,0 +1,1425 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2013-2015, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+  return
+end
+
+local hash = require 'rspamd_cryptobox_hash'
+local rspamd_logger = require 'rspamd_logger'
+local rspamd_util = require 'rspamd_util'
+local rspamd_ip = require "rspamd_ip"
+local fun = require 'fun'
+local lua_util = require 'lua_util'
+local selectors = require "lua_selectors"
+local bit = require 'bit'
+local lua_maps = require "lua_maps"
+local rbl_common = require "plugins/rbl"
+local rspamd_url = require "rspamd_url"
+
+-- This plugin implements various types of RBL checks
+-- Documentation can be found here:
+-- https://rspamd.com/doc/modules/rbl.html
+
+local E = {}
+local N = 'rbl'
+
+-- Checks that could be performed by rbl module
+local local_exclusions
+local white_symbols = {}
+local black_symbols = {}
+local monitored_addresses = {}
+local known_selectors = {} -- map from selector string to selector id
+local url_flag_bits = rspamd_url.flags
+
+local function get_monitored(rbl)
+  local function is_random_monitored()
+    -- Explicit definition
+    if type(rbl.random_monitored) == 'boolean' then
+      return rbl.random_monitored
+    end
+
+    -- We check 127.0.0.1 for merely RBLs with `from` or `received` and only if
+    -- they don't have `no_ip` attribute at the same time
+    --
+    -- Convert to a boolean variable using the common idiom
+    return (not (rbl.from or rbl.received)
+        or rbl.no_ip)
+        and true or false
+  end
+
+  local default_monitored = '1.0.0.127'
+  local ret = {
+    rcode = 'nxdomain',
+    prefix = default_monitored,
+    random = is_random_monitored(),
+  }
+
+  if rbl.monitored_address then
+    ret.prefix = rbl.monitored_address
+  end
+
+  lua_util.debugm(N, rspamd_config,
+      'added monitored address: %s (%s random)',
+      ret.prefix, ret.random)
+
+  return ret
+end
+
+local function validate_dns(lstr)
+  if lstr:match('%.%.') then
+    -- two dots in a row
+    return false, "two dots in a row"
+  end
+  if not rspamd_util.is_valid_utf8(lstr) then
+    -- invalid utf8 detected
+    return false, "invalid utf8"
+  end
+  for v in lstr:gmatch('[^%.]+') do
+    if v:len() > 63 then
+      -- too long label
+      return false, "too long label"
+    end
+    if v:match('^-') or v:match('-$') then
+      -- dash at the beginning or end of label
+      return false, "dash at the beginning or end of label"
+    end
+  end
+  return true
+end
+
+local function maybe_make_hash(data, rule)
+  if rule.hash then
+    local h = hash.create_specific(rule.hash, data)
+    local s
+    if rule.hash_format then
+      if rule.hash_format == 'base32' then
+        s = h:base32()
+      elseif rule.hash_format == 'base64' then
+        s = h:base64()
+      else
+        s = h:hex()
+      end
+    else
+      s = h:hex()
+    end
+
+    if rule.hash_len then
+      s = s:sub(1, rule.hash_len)
+    end
+
+    return s
+  else
+    return data
+  end
+end
+
+local function is_excluded_ip(rip)
+  if local_exclusions and local_exclusions:get_key(rip) then
+    return true
+  end
+  return false
+end
+
+local function ip_to_rbl(ip)
+  return table.concat(ip:inversed_str_octets(), '.')
+end
+
+local function gen_check_rcvd_conditions(rbl, received_total)
+  local min_pos = tonumber(rbl.received_min_pos)
+  local max_pos = tonumber(rbl.received_max_pos)
+  local match_flags = rbl.received_flags
+  local nmatch_flags = rbl.received_nflags
+
+  local function basic_received_check(rh)
+    if not (rh.real_ip and rh.real_ip:is_valid()) then
+      return false
+    end
+    if ((rh.real_ip:get_version() == 6 and rbl.ipv6) or
+        (rh.real_ip:get_version() == 4 and rbl.ipv4)) and
+        ((rbl.exclude_local and not rh.real_ip:is_local() or is_excluded_ip(rh.real_ip)) or not rbl.exclude_local) then
+      return true
+    else
+      return false
+    end
+  end
+
+  local function positioned_received_check(rh, pos)
+    if not rh or not basic_received_check(rh) then
+      return false
+    end
+    local got_flags = rh.flags or E
+    if min_pos then
+      if min_pos < 0 then
+        if min_pos == -1 then
+          if (pos ~= received_total) then
+            return false
+          end
+        else
+          if pos <= (received_total - math.abs(min_pos)) then
+            return false
+          end
+        end
+      elseif pos < min_pos then
+        return false
+      end
+    end
+    if max_pos then
+      if max_pos < -1 then
+        if (received_total - math.abs(max_pos)) >= pos then
+          return false
+        end
+      elseif max_pos > 0 then
+        if pos > max_pos then
+          return false
+        end
+      end
+    end
+    if match_flags then
+      for _, flag in ipairs(match_flags) do
+        if not got_flags[flag] then
+          return false
+        end
+      end
+    end
+    if nmatch_flags then
+      for _, flag in ipairs(nmatch_flags) do
+        if got_flags[flag] then
+          return false
+        end
+      end
+    end
+    return true
+  end
+
+  if not (max_pos or min_pos or match_flags or nmatch_flags) then
+    return basic_received_check
+  else
+    return positioned_received_check
+  end
+end
+
+local matchers = {}
+
+matchers.radix = function(_, _, real_ip, map)
+  return map and map:get_key(real_ip) or false
+end
+
+matchers.equality = function(codes, to_match)
+  if type(codes) ~= 'table' then return codes == to_match end
+  for _, ip in ipairs(codes) do
+    if to_match == ip then
+      return true
+    end
+  end
+  return false
+end
+
+matchers.luapattern = function(codes, to_match)
+  if type(codes) ~= 'table' then
+    return string.find(to_match, '^' .. codes .. '$') and true or false
+  end
+  for _, pattern in ipairs(codes) do
+    if string.find(to_match, '^' .. pattern .. '$') then
+      return true
+    end
+  end
+  return false
+end
+
+matchers.regexp = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
+end
+
+matchers.glob = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
+end
+
+local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
+  local function make_option(ip, label)
+    if ip then
+      return string.format('%s:%s:%s',
+          resolve_table_elt.orig,
+          label,
+          ip)
+    else
+      return string.format('%s:%s',
+          resolve_table_elt.orig,
+          label)
+    end
+  end
+
+  local function insert_result(s, ip, label)
+    if rbl.symbols_prefixes then
+      local prefix = rbl.symbols_prefixes[label]
+
+      if not prefix then
+        rspamd_logger.warnx(task, 'unlisted symbol prefix for %s', label)
+        task:insert_result(s, 1.0, make_option(ip, label))
+      else
+        task:insert_result(prefix .. '_' .. s, 1.0, make_option(ip, label))
+      end
+    else
+      task:insert_result(s, 1.0, make_option(ip, label))
+    end
+  end
+
+  local function insert_results(s, ip)
+    for label in pairs(resolve_table_elt.what) do
+      insert_result(s, ip, label)
+    end
+  end
+
+  if err and (err ~= 'requested record is not found' and
+      err ~= 'no records with this name') then
+    rspamd_logger.infox(task, 'error looking up %s: %s', to_resolve, err)
+    task:insert_result(rbl.symbol .. '_FAIL', 1, string.format('%s:%s',
+        resolve_table_elt.orig, err))
+    return
+  end
+
+  if not results then
+    lua_util.debugm(N, task,
+        'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+        to_resolve, false, err, rbl.symbol)
+    return
+  else
+    lua_util.debugm(N, task,
+        'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+        to_resolve, true, err, rbl.symbol)
+  end
+
+  if rbl.returncodes == nil and rbl.returnbits == nil and rbl.symbol ~= nil then
+    insert_results(rbl.symbol)
+    return
+  end
+
+  local returncodes_maps = rbl.returncodes_maps or {}
+
+  for _, result in ipairs(results) do
+    local ipstr = result:to_string()
+    lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
+    local foundrc = false
+    -- Check return codes
+    if rbl.returnbits then
+      local ipnum = result:to_number()
+      for s, bits in pairs(rbl.returnbits) do
+        for _, check_bit in ipairs(bits) do
+          if bit.band(ipnum, check_bit) == check_bit then
+            foundrc = true
+            insert_results(s)
+            -- Here, we continue with other bits
+          end
+        end
+      end
+    elseif rbl.returncodes then
+      for s, codes in pairs(rbl.returncodes) do
+        local res = match(codes, ipstr, result, returncodes_maps[s])
+        if res then
+          foundrc = true
+          insert_results(s)
+        end
+      end
+    end
+
+    if not foundrc then
+      if rbl.unknown and rbl.symbol then
+        insert_results(rbl.symbol, ipstr)
+      else
+        lua_util.debugm(N, task, '%1 returned unknown result: %2',
+            to_resolve, ipstr)
+      end
+    end
+  end
+
+end
+
+local function gen_rbl_callback(rule)
+  local function is_whitelisted(task, req, req_str, whitelist, what)
+    if rule.ignore_whitelist then
+      lua_util.debugm(N, task,
+          'ignore whitelisting checks to %s by %s: ignore whitelist is being set',
+          req_str, rule.symbol)
+      return false
+    end
+
+    if rule.whitelist then
+      if rule.whitelist:get_key(req) then
+        lua_util.debugm(N, task,
+            'whitelisted %s on %s',
+            req_str, rule.symbol)
+
+        return true
+      end
+    end
+
+    -- Maybe whitelisted by some other rbl rule
+    if whitelist then
+      local wl = whitelist[req_str]
+      if wl then
+        lua_util.debugm(N, task,
+            'whitelisted request to %s by %s (%s) rbl rule (%s checked type, %s whitelist type)',
+            req_str, wl.type, wl.symbol, what, wl.type)
+        if wl.type == what then
+          -- This was decided to be a bad idea as in case of whitelisting a request to blacklist
+          -- is not even sent
+          --task:adjust_result(wl.symbol, 0.0 / 0.0, rule.symbol)
+
+          return true
+        end
+      end
+    end
+
+    return false
+  end
+
+  local function add_dns_request(task, req, forced, is_ip, requests_table, label, whitelist)
+    local req_str = req
+    if is_ip then
+      req_str = tostring(req)
+    end
+
+    if whitelist and is_whitelisted(task, req, req_str, whitelist, label) then
+      return
+    end
+
+    if is_ip then
+      req = ip_to_rbl(req)
+    end
+
+    if requests_table[req] then
+      -- Duplicate request
+      local nreq = requests_table[req]
+      if forced and not nreq.forced then
+        nreq.forced = true
+      end
+      if not nreq.what[label] then
+        nreq.what[label] = true
+      end
+
+      return true, nreq -- Duplicate
+    else
+      local nreq
+
+      local resolve_ip = rule.resolve_ip and not is_ip
+      if rule.process_script then
+        local processed = rule.process_script(req, rule.rbl, task, resolve_ip)
+
+        if processed then
+          nreq = {
+            forced = forced,
+            n = processed,
+            orig = req_str,
+            resolve_ip = resolve_ip,
+            what = { [label] = true },
+          }
+          requests_table[req] = nreq
+        end
+      else
+        local to_resolve
+        local origin = req
+
+        if not resolve_ip then
+          origin = maybe_make_hash(req, rule)
+          to_resolve = string.format('%s.%s',
+              origin,
+              rule.rbl)
+        else
+          -- First, resolve origin stuff without hashing or anything
+          to_resolve = origin
+        end
+
+        nreq = {
+          forced = forced,
+          n = to_resolve,
+          orig = req_str,
+          resolve_ip = resolve_ip,
+          what = { [label] = true },
+        }
+        requests_table[req] = nreq
+      end
+      return false, nreq
+    end
+  end
+
+  -- Here, we have functional approach: we form a pipeline of functions
+  -- f1, f2, ... fn. Each function accepts task and return boolean value
+  -- that allows to process pipeline further
+  -- Each function in the pipeline can add something to `dns_req` vector as a side effect
+  local function is_alive(_, _)
+    if rule.monitored then
+      if not rule.monitored:alive() then
+        return false
+      end
+    end
+
+    return true
+  end
+
+  local function check_required_symbols(task, _)
+    if rule.require_symbols then
+      return fun.all(function(sym)
+        task:has_symbol(sym)
+      end, rule.require_symbols)
+    end
+
+    return true
+  end
+
+  local function check_user(task, _)
+    if task:get_user() then
+      return false
+    end
+
+    return true
+  end
+
+  local function check_local(task, _)
+    local ip = task:get_from_ip()
+
+    if ip and not ip:is_valid() then
+      ip = nil
+    end
+
+    if ip and ip:is_local() or is_excluded_ip(ip) then
+      return false
+    end
+
+    return true
+  end
+
+  local function check_helo(task, requests_table, whitelist)
+    local helo = task:get_helo()
+
+    if not helo then
+      -- Avoid pipeline breaking
+      return true
+    end
+
+    add_dns_request(task, helo, true, false, requests_table,
+        'helo', whitelist)
+
+    return true
+  end
+
+  local function check_dkim(task, requests_table, whitelist)
+    local das = task:get_symbol('DKIM_TRACE')
+    local mime_from_domain
+
+    if das and das[1] and das[1].options then
+
+      if rule.dkim_match_from then
+        -- We check merely mime from
+        mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain
+        if mime_from_domain then
+          local mime_from_domain_tld = rule.url_full_hostname and
+              mime_from_domain or rspamd_util.get_tld(mime_from_domain)
+
+          if rule.url_compose_map then
+            mime_from_domain = rule.url_compose_map:process_url(task, mime_from_domain_tld, mime_from_domain)
+          else
+            mime_from_domain = mime_from_domain_tld
+          end
+        end
+      end
+
+      for _, d in ipairs(das[1].options) do
+
+        local domain, result = d:match('^([^%:]*):([%+%-%~])$')
+
+        -- We must ignore bad signatures, omg
+        if domain and result and result == '+' then
+          if rule.dkim_match_from then
+            -- We check merely mime from
+            local domain_tld = domain
+            if not rule.dkim_domainonly then
+              -- Adjust
+              domain_tld = rspamd_util.get_tld(domain)
+
+              if rule.url_compose_map then
+                domain_tld = rule.url_compose_map:process_url(task, domain_tld, domain)
+              elseif rule.url_full_hostname then
+                domain_tld = domain
+              end
+            end
+
+            if mime_from_domain and mime_from_domain == domain_tld then
+              add_dns_request(task, domain_tld, true, false, requests_table,
+                  'dkim', whitelist)
+            end
+          else
+            if rule.dkim_domainonly then
+              local domain_tld = rspamd_util.get_tld(domain)
+              if rule.url_compose_map then
+                domain_tld = rule.url_compose_map:process_url(task, domain_tld, domain)
+              elseif rule.url_full_hostname then
+                domain_tld = domain
+              end
+              add_dns_request(task, domain_tld,
+                  false, false, requests_table, 'dkim', whitelist)
+            else
+              add_dns_request(task, domain, false, false, requests_table,
+                  'dkim', whitelist)
+            end
+          end
+        end
+      end
+    end
+
+    return true
+  end
+
+  local function check_urls(task, requests_table, whitelist)
+    local esld_lim = 1
+
+    if rule.url_compose_map then
+      esld_lim = nil -- Avoid esld limit as we use custom composition rules
+    end
+    local ex_params = {
+      task = task,
+      limit = rule.requests_limit,
+      ignore_redirected = true,
+      ignore_ip = rule.no_ip,
+      need_images = rule.images,
+      need_emails = false,
+      need_content = rule.content_urls or false,
+      esld_limit = esld_lim,
+      no_cache = true,
+    }
+
+    if rule.numeric_urls then
+      if rule.content_urls then
+        if not rule.images then
+          ex_params.flags_mode = 'explicit'
+          ex_params.flags = { 'numeric' }
+          ex_params.filter = function(url)
+            return (bit.band(url:get_flags_num(), url_flag_bits.image) == 0)
+          end
+        else
+          ex_params.filter = function(url)
+            return (bit.band(url:get_flags_num(), url_flag_bits.numeric) ~= 0)
+          end
+        end
+      elseif rule.images then
+        ex_params.filter = function(url)
+          return (bit.band(url:get_flags_num(), url_flag_bits.numeric) ~= 0)
+        end
+      else
+        ex_params.flags_mode = 'explicit'
+        ex_params.flags = { 'numeric' }
+        ex_params.filter = function(url)
+          return (bit.band(url:get_flags_num(), url_flag_bits.content) == 0)
+        end
+      end
+    elseif not rule.urls and (rule.content_urls or rule.images) then
+      ex_params.flags_mode = 'explicit'
+      ex_params.flags = {}
+      if rule.content_urls then
+        table.insert(ex_params.flags, 'content')
+      end
+      if rule.images then
+        table.insert(ex_params.flags, 'image')
+      end
+    end
+
+    local urls = lua_util.extract_specific_urls(ex_params)
+
+    for _, u in ipairs(urls) do
+      local flags = u:get_flags_num()
+
+      if bit.band(flags, url_flag_bits.numeric) ~= 0 then
+        -- For numeric urls we convert data to the ip address and
+        -- reverse octets. See #3948 for details
+        local to_resolve = u:get_host()
+        local addr = rspamd_ip.from_string(to_resolve)
+
+        if addr then
+          to_resolve = table.concat(addr:inversed_str_octets(), ".")
+        end
+        add_dns_request(task, to_resolve, false,
+            false, requests_table, 'url', whitelist)
+      else
+        local url_hostname = u:get_host()
+        local url_tld = rule.url_full_hostname and url_hostname or u:get_tld()
+        if rule.url_compose_map then
+          url_tld = rule.url_compose_map:process_url(task, url_tld, url_hostname)
+        end
+        add_dns_request(task, url_tld, false,
+            false, requests_table, 'url', whitelist)
+      end
+    end
+
+    return true
+  end
+
+  local function check_from(task, requests_table, whitelist)
+    local ip = task:get_from_ip()
+
+    if not ip or not ip:is_valid() then
+      return true
+    end
+    if (ip:get_version() == 6 and rule.ipv6) or
+        (ip:get_version() == 4 and rule.ipv4) then
+      add_dns_request(task, ip, true, true,
+          requests_table, 'from',
+          whitelist)
+    end
+
+    return true
+  end
+
+  local function check_received(task, requests_table, whitelist)
+    local received = fun             .filter(function(h)
+      return not h['flags']['artificial']
+    end, task:get_received_headers()):totable()
+
+    local received_total = #received
+    local check_conditions = gen_check_rcvd_conditions(rule, received_total)
+
+    for pos, rh in ipairs(received) do
+      if check_conditions(rh, pos) then
+        add_dns_request(task, rh.real_ip, false, true,
+            requests_table, 'received',
+            whitelist)
+      end
+    end
+
+    return true
+  end
+
+  local function check_rdns(task, requests_table, whitelist)
+    local hostname = task:get_hostname()
+    if hostname == nil or hostname == 'unknown' then
+      return true
+    end
+
+    add_dns_request(task, hostname, true, false,
+        requests_table, 'rdns', whitelist)
+
+    return true
+  end
+
+  local function check_selector(task, requests_table, whitelist)
+    for selector_label, selector in pairs(rule.selectors) do
+      local res = selector(task)
+
+      if res and type(res) == 'table' then
+        for _, r in ipairs(res) do
+          add_dns_request(task, r, false, false, requests_table,
+              selector_label, whitelist)
+        end
+      elseif res then
+        add_dns_request(task, res, false, false,
+            requests_table, selector_label, whitelist)
+      end
+    end
+
+    return true
+  end
+
+  local function check_email_table(task, email_tbl, requests_table, whitelist, what)
+    lua_util.remove_email_aliases(email_tbl)
+    email_tbl.domain = email_tbl.domain:lower()
+    email_tbl.user = email_tbl.user:lower()
+
+    if email_tbl.domain == '' or email_tbl.user == '' then
+      rspamd_logger.infox(task, "got an email with some empty parts: '%s@%s'; skip it in the checks",
+          email_tbl.user, email_tbl.domain)
+      return
+    end
+
+    if rule.emails_domainonly then
+      add_dns_request(task, email_tbl.domain, false, false, requests_table,
+          what, whitelist)
+    else
+      -- Also check WL for domain only
+      if is_whitelisted(task,
+          email_tbl.domain,
+          email_tbl.domain,
+          whitelist,
+          what) then
+        return
+      end
+      local delimiter = '.'
+      if rule.emails_delimiter then
+        delimiter = rule.emails_delimiter
+      else
+        if rule.hash then
+          delimiter = '@'
+        end
+      end
+      add_dns_request(task, string.format('%s%s%s',
+          email_tbl.user, delimiter, email_tbl.domain), false, false,
+          requests_table, what, whitelist)
+    end
+  end
+
+  local function check_emails(task, requests_table, whitelist)
+    local ex_params = {
+      task = task,
+      limit = rule.requests_limit,
+      filter = function(u)
+        return u:get_protocol() == 'mailto'
+      end,
+      need_emails = true,
+      prefix = 'rbl_email'
+    }
+
+    if rule.emails_domainonly then
+      if not rule.url_compose_map then
+        ex_params.esld_limit = 1
+      end
+      ex_params.prefix = 'rbl_email_domainonly'
+    end
+
+    local emails = lua_util.extract_specific_urls(ex_params)
+
+    for _, email in ipairs(emails) do
+      local domain
+      if rule.emails_domainonly and not rule.url_full_hostname then
+        if rule.url_compose_map then
+          domain = rule.url_compose_map:process_url(task, email:get_tld(), email:get_host())
+        else
+          domain = email:get_tld()
+        end
+      else
+        domain = email:get_host()
+      end
+
+      local email_tbl = {
+        domain = domain or '',
+        user = email:get_user() or '',
+        addr = tostring(email),
+      }
+      check_email_table(task, email_tbl, requests_table, whitelist, 'email')
+    end
+
+    return true
+  end
+
+  local function check_replyto(task, requests_table, whitelist)
+    local function get_raw_header(name)
+      return ((task:get_header_full(name) or {})[1] or {})['value']
+    end
+
+    local replyto = get_raw_header('Reply-To')
+    if replyto then
+      local rt = rspamd_util.parse_mail_address(replyto, task:get_mempool())
+      lua_util.debugm(N, task, 'check replyto %s', rt[1])
+
+      if rt and rt[1] and (rt[1].addr and #rt[1].addr > 0) then
+        check_email_table(task, rt[1], requests_table, whitelist, 'replyto')
+      end
+    end
+
+    return true
+  end
+
+  -- Create function pipeline depending on rbl settings
+  local pipeline = {
+    is_alive, -- check monitored status
+    check_required_symbols -- if we have require_symbols then check those symbols
+  }
+  local description = {
+    'alive',
+  }
+
+  if rule.exclude_users then
+    pipeline[#pipeline + 1] = check_user
+    description[#description + 1] = 'user'
+  end
+
+  if rule.exclude_local then
+    pipeline[#pipeline + 1] = check_local
+    description[#description + 1] = 'local'
+  end
+
+  if rule.helo then
+    pipeline[#pipeline + 1] = check_helo
+    description[#description + 1] = 'helo'
+  end
+
+  if rule.dkim then
+    pipeline[#pipeline + 1] = check_dkim
+    description[#description + 1] = 'dkim'
+  end
+
+  if rule.emails then
+    pipeline[#pipeline + 1] = check_emails
+    description[#description + 1] = 'emails'
+  end
+  if rule.replyto then
+    pipeline[#pipeline + 1] = check_replyto
+    description[#description + 1] = 'replyto'
+  end
+
+  if rule.urls or rule.content_urls or rule.images or rule.numeric_urls then
+    pipeline[#pipeline + 1] = check_urls
+    description[#description + 1] = 'urls'
+  end
+
+  if rule.from then
+    pipeline[#pipeline + 1] = check_from
+    description[#description + 1] = 'ip'
+  end
+
+  if rule.received then
+    pipeline[#pipeline + 1] = check_received
+    description[#description + 1] = 'received'
+  end
+
+  if rule.rdns then
+    pipeline[#pipeline + 1] = check_rdns
+    description[#description + 1] = 'rdns'
+  end
+
+  if rule.selector then
+    pipeline[#pipeline + 1] = check_selector
+    description[#description + 1] = 'selector'
+  end
+
+  if not rule.returncodes_matcher then
+    rule.returncodes_matcher = 'equality'
+  end
+  local match = matchers[rule.returncodes_matcher]
+
+  local callback_f = function(task)
+    -- DNS requests to issue (might be hashed afterwards)
+    local dns_req = {}
+    local whitelist = task:cache_get('rbl_whitelisted') or {}
+
+    local function gen_rbl_dns_callback(resolve_table_elt)
+      return function(_, to_resolve, results, err)
+        rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt, match)
+      end
+    end
+
+    -- Execute functions pipeline
+    for i, f in ipairs(pipeline) do
+      if not f(task, dns_req, whitelist) then
+        lua_util.debugm(N, task,
+            "skip rbl check: %s; pipeline condition %s returned false",
+            rule.symbol, i)
+        return
+      end
+    end
+
+    -- Now check all DNS requests pending and emit them
+    local r = task:get_resolver()
+    -- Used for 2 passes ip resolution
+    local resolved_req = {}
+    local nresolved = 0
+
+    -- This is called when doing resolve_ip phase...
+    local function gen_rbl_ip_dns_callback(orig_resolve_table_elt)
+      return function(_, _, results, err)
+        if not err then
+          for _, dns_res in ipairs(results) do
+            -- Check if we have rspamd{ip} userdata
+            if type(dns_res) == 'userdata' then
+              -- Add result as an actual RBL request
+              local label = next(orig_resolve_table_elt.what)
+              local dup, nreq = add_dns_request(task, dns_res, false, true,
+                  resolved_req, label)
+              -- Add original name
+              if not dup then
+                nreq.orig = nreq.orig .. ':' .. orig_resolve_table_elt.n
+              end
+            end
+          end
+        end
+
+        nresolved = nresolved - 1
+
+        if nresolved == 0 then
+          -- Emit real RBL requests as there are no ip resolution requests
+          for name, req in pairs(resolved_req) do
+            local val_res, val_error = validate_dns(req.n)
+            if val_res then
+              lua_util.debugm(N, task, "rbl %s; resolve %s -> %s",
+                  rule.symbol, name, req.n)
+              r:resolve_a({
+                task = task,
+                name = req.n,
+                callback = gen_rbl_dns_callback(req),
+                forced = req.forced
+              })
+            else
+              rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s: %s',
+                  req.n, rule.symbol, val_error)
+            end
+          end
+        end
+      end
+    end
+
+    for name, req in pairs(dns_req) do
+      local val_res, val_error = validate_dns(req.n)
+      if val_res then
+        lua_util.debugm(N, task, "rbl %s; resolve %s -> %s",
+            rule.symbol, name, req.n)
+
+        if req.resolve_ip then
+          -- Deal with both ipv4 and ipv6
+          -- Resolve names first
+          if r:resolve_a({
+            task = task,
+            name = req.n,
+            callback = gen_rbl_ip_dns_callback(req),
+            forced = req.forced
+          }) then
+            nresolved = nresolved + 1
+          end
+          if r:resolve('aaaa', {
+            task = task,
+            name = req.n,
+            callback = gen_rbl_ip_dns_callback(req),
+            forced = req.forced
+          }) then
+            nresolved = nresolved + 1
+          end
+        else
+          r:resolve_a({
+            task = task,
+            name = req.n,
+            callback = gen_rbl_dns_callback(req),
+            forced = req.forced
+          })
+        end
+
+      else
+        rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s: %s',
+            req.n, rule.symbol, val_error)
+      end
+    end
+  end
+
+  return callback_f, string.format('checks: %s', table.concat(description, ','))
+end
+
+local map_match_types = {
+  glob = true,
+  radix = true,
+  regexp = true,
+}
+
+local function add_rbl(key, rbl, global_opts)
+  if not rbl.symbol then
+    rbl.symbol = key:upper()
+  end
+
+  local flags_tbl = { 'no_squeeze' }
+  if rbl.is_whitelist then
+    flags_tbl[#flags_tbl + 1] = 'nice'
+  end
+
+  -- Check if rbl is available for empty tasks
+  if not (rbl.emails or rbl.urls or rbl.dkim or rbl.received or rbl.selector or rbl.replyto) or
+      rbl.is_empty then
+    flags_tbl[#flags_tbl + 1] = 'empty'
+  end
+
+  if rbl.selector then
+
+    rbl.selectors = {}
+    if type(rbl.selector) ~= 'table' then
+      rbl.selector = { ['selector'] = rbl.selector }
+    end
+
+    for selector_label, selector in pairs(rbl.selector) do
+      if known_selectors[selector] then
+        lua_util.debugm(N, rspamd_config, 'reuse selector id %s',
+            known_selectors[selector].id)
+        rbl.selectors[selector_label] = known_selectors[selector].selector
+      else
+
+        if type(rbl.selector_flatten) ~= 'boolean' then
+          -- Fail-safety
+          rbl.selector_flatten = true
+        end
+        local sel = selectors.create_selector_closure(rspamd_config, selector, '',
+            rbl.selector_flatten)
+
+        if not sel then
+          rspamd_logger.errx('invalid selector for rbl rule %s: %s', key, selector)
+          return false
+        end
+
+        rbl.selector = sel
+        known_selectors[selector] = {
+          selector = sel,
+          id = #lua_util.keys(known_selectors) + 1,
+        }
+        rbl.selectors[selector_label] = known_selectors[selector].selector
+      end
+    end
+
+  end
+
+  if rbl.process_script then
+    local ret, f = lua_util.callback_from_string(rbl.process_script)
+
+    if ret then
+      rbl.process_script = f
+    else
+      rspamd_logger.errx(rspamd_config,
+          'invalid process script for rbl rule %s: %s; %s',
+          key, rbl.process_script, f)
+      return false
+    end
+  end
+
+  if rbl.whitelist then
+    local def_type = 'set'
+    if rbl.from or rbl.received then
+      def_type = 'radix'
+    end
+    rbl.whitelist = lua_maps.map_add_from_ucl(rbl.whitelist, def_type,
+        'RBL whitelist for ' .. rbl.symbol)
+    rspamd_logger.infox(rspamd_config, 'added %s whitelist for RBL %s',
+        def_type, rbl.symbol)
+  end
+
+  local match_type = rbl.returncodes_matcher
+  if match_type and rbl.returncodes and map_match_types[match_type] then
+    if not rbl.returncodes_maps then
+      rbl.returncodes_maps = {}
+    end
+    for label, v in pairs(rbl.returncodes) do
+      if type(v) ~= 'table' then
+        v = {v}
+      end
+      rbl.returncodes_maps[label] = lua_maps.map_add_from_ucl(v, match_type, string.format('%s_%s RBL returncodes', label, rbl.symbol))
+    end
+  end
+
+  if rbl.url_compose_map then
+    local lua_urls_compose = require "lua_urls_compose"
+    rbl.url_compose_map = lua_urls_compose.add_composition_map(rspamd_config, rbl.url_compose_map)
+
+    if rbl.url_compose_map then
+      rspamd_logger.infox(rspamd_config, 'added url composition map for RBL %s',
+          rbl.symbol)
+    end
+  end
+
+  if not rbl.whitelist and not rbl.ignore_url_whitelist and (global_opts.url_whitelist or rbl.url_whitelist) and
+      (rbl.urls or rbl.emails or rbl.dkim or rbl.replyto) and
+      not (rbl.from or rbl.received) then
+    local def_type = 'set'
+    rbl.whitelist = lua_maps.map_add_from_ucl(rbl.url_whitelist or global_opts.url_whitelist, def_type,
+        'RBL url whitelist for ' .. rbl.symbol)
+    rspamd_logger.infox(rspamd_config, 'added URL whitelist for RBL %s',
+        rbl.symbol)
+  end
+
+  local callback, description = gen_rbl_callback(rbl)
+
+  if callback then
+    local id
+
+    if rbl.symbols_prefixes then
+      id = rspamd_config:register_symbol {
+        type = 'callback',
+        callback = callback,
+        groups = { 'rbl' },
+        name = rbl.symbol .. '_CHECK',
+        flags = table.concat(flags_tbl, ',')
+      }
+
+      for _, prefix in pairs(rbl.symbols_prefixes) do
+        -- For unknown results...
+        rspamd_config:register_symbol {
+          type = 'virtual',
+          parent = id,
+          group = 'rbl',
+          score = 0,
+          name = prefix .. '_' .. rbl.symbol,
+        }
+      end
+      if not (rbl.is_whitelist or rbl.ignore_whitelist) then
+        table.insert(black_symbols, rbl.symbol .. '_CHECK')
+      else
+        lua_util.debugm(N, rspamd_config, 'rule %s ignores whitelists: rbl.is_whitelist = %s, ' ..
+            'rbl.ignore_whitelist = %s',
+            rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist)
+      end
+    else
+      id = rspamd_config:register_symbol {
+        type = 'callback',
+        callback = callback,
+        name = rbl.symbol,
+        groups = { 'rbl' },
+        group = 'rbl',
+        score = 0,
+        flags = table.concat(flags_tbl, ',')
+      }
+      if not (rbl.is_whitelist or rbl.ignore_whitelist) then
+        table.insert(black_symbols, rbl.symbol)
+      else
+        lua_util.debugm(N, rspamd_config, 'rule %s ignores whitelists: rbl.is_whitelist = %s, ' ..
+            'rbl.ignore_whitelist = %s',
+            rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist)
+      end
+    end
+
+    rspamd_logger.infox(rspamd_config, 'added rbl rule %s: %s',
+        rbl.symbol, description)
+    lua_util.debugm(N, rspamd_config, 'rule dump for %s: %s',
+        rbl.symbol, rbl)
+
+    local check_sym = rbl.symbols_prefixes and rbl.symbol .. '_CHECK' or rbl.symbol
+
+    if rbl.dkim then
+      rspamd_config:register_dependency(check_sym, 'DKIM_CHECK')
+    end
+
+    if rbl.require_symbols then
+      for _, dep in ipairs(rbl.require_symbols) do
+        rspamd_config:register_dependency(check_sym, dep)
+      end
+    end
+
+    -- Failure symbol
+    rspamd_config:register_symbol {
+      type = 'virtual',
+      flags = 'nostat',
+      name = rbl.symbol .. '_FAIL',
+      parent = id,
+      score = 0.0,
+    }
+
+    local function process_return_code(suffix)
+      local function process_specific_suffix(s)
+        if s ~= rbl.symbol then
+          -- hack
+
+          rspamd_config:register_symbol {
+            type = 'virtual',
+            parent = id,
+            name = s,
+            group = 'rbl',
+            score = 0,
+          }
+        end
+        if rbl.is_whitelist then
+          if rbl.whitelist_exception then
+            local found_exception = false
+            for _, e in ipairs(rbl.whitelist_exception) do
+              if e == s then
+                found_exception = true
+                break
+              end
+            end
+            if not found_exception then
+              table.insert(white_symbols, s)
+            end
+          else
+            table.insert(white_symbols, s)
+          end
+        else
+          if not rbl.ignore_whitelist then
+            table.insert(black_symbols, s)
+          end
+        end
+      end
+
+      if rbl.symbols_prefixes then
+        for _, prefix in pairs(rbl.symbols_prefixes) do
+          process_specific_suffix(prefix .. '_' .. suffix)
+        end
+      else
+        process_specific_suffix(suffix)
+      end
+
+    end
+
+    if rbl.returncodes then
+      for s, _ in pairs(rbl.returncodes) do
+        process_return_code(s)
+      end
+    end
+
+    if rbl.returnbits then
+      for s, _ in pairs(rbl.returnbits) do
+        process_return_code(s)
+      end
+    end
+
+    -- Process monitored
+    if not rbl.disable_monitoring then
+      if not monitored_addresses[rbl.rbl] then
+        monitored_addresses[rbl.rbl] = true
+        rbl.monitored = rspamd_config:register_monitored(rbl.rbl, 'dns',
+            get_monitored(rbl))
+      end
+    end
+    return true
+  end
+
+  return false
+end
+
+-- Configuration
+local opts = rspamd_config:get_all_opt(N)
+if not (opts and type(opts) == 'table') then
+  rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+  lua_util.disable_module(N, "config")
+  return
+end
+
+-- Plugin defaults should not be changed - override these in config
+-- New defaults should not alter behaviour
+
+
+opts = lua_util.override_defaults(rbl_common.default_options, opts)
+
+if opts.rules and opts.rbls then
+  -- Common issue :(
+  rspamd_logger.infox(rspamd_config, 'merging `rules` and `rbls` keys for compatibility')
+  opts.rbls = lua_util.override_defaults(opts.rbls, opts.rules)
+end
+
+if (opts['local_exclude_ip_map'] ~= nil) then
+  local_exclusions = lua_maps.map_add(N, 'local_exclude_ip_map', 'radix',
+      'RBL exclusions map')
+end
+
+-- TODO: this code should be universal for all modules that use selectors to allow
+-- maps usage from selectors registered for a specific module
+if type(opts.attached_maps) == 'table' then
+  opts.attached_maps_processed = {}
+  for i, map in ipairs(opts.attached_maps) do
+    -- Store maps in the configuration table to keep lifetime track
+    opts.attached_maps_processed[i] = lua_maps.map_add_from_ucl(map)
+    if opts.attached_maps_processed[i] == nil then
+      rspamd_logger.warnx(rspamd_config, "cannot parse attached map: %s", map)
+    end
+  end
+end
+
+for key, rbl in pairs(opts.rbls) do
+  if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then
+    rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key)
+  else
+    -- Aliases
+    if type(rbl.ignore_default) == 'boolean' then
+      rbl.ignore_defaults = rbl.ignore_default
+    end
+    if type(rbl.ignore_whitelists) == 'boolean' then
+      rbl.ignore_whitelist = rbl.ignore_whitelists
+    end
+    -- Propagate default options from opts to rule
+    if not rbl.ignore_defaults then
+      for default_opt_key, _ in pairs(rbl_common.default_options) do
+        local rbl_opt = default_opt_key:sub(#('default_') + 1)
+        if rbl[rbl_opt] == nil then
+          rbl[rbl_opt] = opts[default_opt_key]
+        end
+      end
+    end
+
+    if not rbl.requests_limit then
+      rbl.requests_limit = rspamd_config:get_dns_max_requests()
+    end
+
+    local res, err = rbl_common.rule_schema:transform(rbl)
+    if not res then
+      rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
+          key, err)
+    else
+      res = rbl_common.convert_checks(res, rbl.symbol or key:upper())
+      -- Aliases
+      if res.return_codes then
+        res.returncodes = res.return_codes
+      end
+      if res.return_bits then
+        res.returnbits = res.return_bits
+      end
+
+      if not res then
+        rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
+            key, err)
+      else
+        add_rbl(key, res, opts)
+      end
+    end
+  end -- rbl.enabled
+end
+
+-- We now create two symbols:
+-- * RBL_CALLBACK_WHITE that depends on all symbols white
+-- * RBL_CALLBACK that depends on all symbols black to participate in depends chains
+local function rbl_callback_white(task)
+  local whitelisted_elements = {}
+  for _, w in ipairs(white_symbols) do
+    local ws = task:get_symbol(w)
+    if ws and ws[1] then
+      ws = ws[1]
+      if not ws.options then
+        ws.options = {}
+      end
+      for _, opt in ipairs(ws.options) do
+        local elt, what = opt:match('^([^:]+):([^:]+)')
+        lua_util.debugm(N, task, 'found whitelist from %s: %s(%s)', w,
+            elt, what)
+        if elt and what then
+          whitelisted_elements[elt] = {
+            type = what,
+            symbol = w,
+          }
+        end
+      end
+    end
+  end
+
+  task:cache_set('rbl_whitelisted', whitelisted_elements)
+
+  lua_util.debugm(N, task, "finished rbl whitelists processing")
+end
+
+local function rbl_callback_fin(task)
+  -- Do nothing
+  lua_util.debugm(N, task, "finished rbl processing")
+end
+
+rspamd_config:register_symbol {
+  type = 'callback',
+  callback = rbl_callback_white,
+  name = 'RBL_CALLBACK_WHITE',
+  flags = 'nice,empty,no_squeeze',
+  groups = { 'rbl' },
+  augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
+}
+
+rspamd_config:register_symbol {
+  type = 'callback',
+  callback = rbl_callback_fin,
+  name = 'RBL_CALLBACK',
+  flags = 'empty,no_squeeze',
+  groups = { 'rbl' },
+  augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
+}
+
+for _, w in ipairs(white_symbols) do
+  rspamd_config:register_dependency('RBL_CALLBACK_WHITE', w)
+end
+
+for _, b in ipairs(black_symbols) do
+  rspamd_config:register_dependency(b, 'RBL_CALLBACK_WHITE')
+  rspamd_config:register_dependency('RBL_CALLBACK', b)
+end