Adding upstream version 5.7.1.upstream/5.7.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 498 insertions, 0 deletions

diff --git a/modules/prefill/.packaging/test.config b/modules/prefill/.packaging/test.config
new file mode 100644
index 0000000..d0258b0
--- /dev/null
+++ b/modules/prefill/.packaging/test.config
@@ -0,0 +1,4 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+modules.load('prefill')
+assert(prefill)
+quit()
diff --git a/modules/prefill/README.rst b/modules/prefill/README.rst
new file mode 100644
index 0000000..a99f1f0
--- /dev/null
+++ b/modules/prefill/README.rst
@@ -0,0 +1,43 @@
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _mod-prefill:
+
+Cache prefilling
+================
+
+This module provides ability to periodically prefill the DNS cache by importing root zone data obtained over HTTPS.
+
+Intended users of this module are big resolver operators which will benefit from decreased latencies and smaller amount of traffic towards DNS root servers.
+
+Example configuration is:
+
+.. code-block:: lua
+
+    modules.load('prefill')
+    prefill.config({
+        ['.'] = {
+            url = 'https://www.internic.net/domain/root.zone',
+            interval = 86400, -- seconds
+            ca_file = '/etc/pki/tls/certs/ca-bundle.crt', -- optional
+        }
+    })
+
+This configuration downloads the zone file from URL `https://www.internic.net/domain/root.zone` and imports it into the cache every 86400 seconds (1 day). The HTTPS connection is authenticated using a CA certificate from file `/etc/pki/tls/certs/ca-bundle.crt` and signed zone content is validated using DNSSEC.
+
+The root zone to be imported must be signed using DNSSEC and the resolver must have a valid DNSSEC configuration.
+
+.. csv-table::
+ :header: "Parameter", "Description"
+
+ "ca_file", "path to CA certificate bundle used to authenticate the HTTPS connection (optional, system-wide store will be used if not specified)"
+ "interval", "number of seconds between zone data refresh attempts"
+ "url", "URL of a file in :rfc:`1035` zone file format"
+
+Only root zone import is supported at the moment.
+
+Dependencies
+------------
+
+Prefilling depends on the lua-http_ library.
+
+.. _lua-http: https://luarocks.org/modules/daurnimator/http
diff --git a/modules/prefill/prefill.lua b/modules/prefill/prefill.lua
new file mode 100644
index 0000000..f4a4288
--- /dev/null
+++ b/modules/prefill/prefill.lua
@@ -0,0 +1,199 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+local ffi = require('ffi')
+
+local rz_url = "https://www.internic.net/domain/root.zone"
+local rz_local_fname = "root.zone"
+local rz_ca_file = nil
+local rz_event_id = nil
+
+local rz_default_interval = 86400
+local rz_https_fail_interval = 600
+local rz_import_error_interval = 600
+local rz_cur_interval = rz_default_interval
+local rz_interval_randomizer_limit = 10
+local rz_interval_threshold = 5
+local rz_interval_min = 3600
+
+local rz_first_try = true
+
+local prefill = {}
+
+-- hack for circular dependency between timer() and fill_cache()
+local forward_references = {}
+
+local function stop_timer()
+	if rz_event_id then
+		event.cancel(rz_event_id)
+		rz_event_id = nil
+	end
+end
+
+local function timer()
+	stop_timer()
+	worker.bg_worker.cq:wrap(forward_references.fill_cache)
+end
+
+local function restart_timer(after)
+	stop_timer()
+	rz_event_id = event.after(after * sec, timer)
+end
+
+local function display_delay(time)
+	local days = math.floor(time / 86400)
+	local hours = math.floor((time % 86400) / 3600)
+	local minutes = math.floor((time % 3600) / 60)
+	local seconds = math.floor(time % 60)
+	if days > 0 then
+		return string.format("%d days %02d hours", days, hours)
+	elseif hours > 0 then
+		return string.format("%02d hours %02d minutes", hours, minutes)
+	elseif minutes > 0 then
+		return string.format("%02d minutes %02d seconds", minutes, seconds)
+	end
+	return string.format("%02d seconds", seconds)
+end
+
+-- returns: number of seconds the file is valid for
+-- 0 indicates immediate download
+local function get_file_ttl(fname)
+	local mtime = tonumber(ffi.C.kr_file_mtime(fname))
+
+	if mtime > 0 then
+		local age = os.time() - mtime
+		return math.max(
+			rz_cur_interval - age,
+			0)
+	else
+		return 0  -- file does not exist, download now
+	end
+end
+
+local function download(url, fname)
+	local kluautil = require('kluautil')
+	local file, rcode, errmsg
+	file, errmsg = io.open(fname, 'w')
+	if not file then
+		error(string.format("[prefil] unable to open file %s (%s)",
+			fname, errmsg))
+	end
+
+	log_info(ffi.C.LOG_GRP_PREFILL, "downloading root zone to file %s ...", fname)
+	rcode, errmsg = kluautil.kr_https_fetch(url, file, rz_ca_file)
+	if rcode == nil then
+		error(string.format("[prefil] fetch of `%s` failed: %s", url, errmsg))
+	end
+
+	file:close()
+end
+
+local function import(fname)
+	local ret = ffi.C.zi_zone_import({
+		zone_file = fname,
+		time_src = ffi.C.ZI_STAMP_MTIM, -- the file might be slightly older
+	})
+	if ret == 0 then
+		log_info(ffi.C.LOG_GRP_PREFILL, "zone successfully parsed, import started")
+	else
+		error(string.format(
+			"[prefil] zone import failed: %s", ffi.C.knot_strerror(ret)
+		))
+	end
+end
+
+function forward_references.fill_cache()
+	local file_ttl = get_file_ttl(rz_local_fname)
+
+	if file_ttl > rz_interval_threshold then
+		log_info(ffi.C.LOG_GRP_PREFILL, "root zone file valid for %s, reusing data from disk",
+			display_delay(file_ttl))
+	else
+		local ok, errmsg = pcall(download, rz_url, rz_local_fname)
+		if not ok then
+			rz_cur_interval = rz_https_fail_interval
+						- math.random(rz_interval_randomizer_limit)
+			log_info(ffi.C.LOG_GRP_PREFILL, "cannot download new zone (%s), "
+				.. "will retry root zone download in %s",
+				errmsg, display_delay(rz_cur_interval))
+			restart_timer(rz_cur_interval)
+			os.remove(rz_local_fname)
+			return
+		end
+		file_ttl = rz_default_interval
+	end
+	-- file is up to date, import
+	-- import/filter function gets executed after resolver/module
+	local ok, errmsg = pcall(import, rz_local_fname)
+	if not ok then
+		if rz_first_try then
+			rz_first_try = false
+			rz_cur_interval = 1
+		else
+			rz_cur_interval = rz_import_error_interval
+				- math.random(rz_interval_randomizer_limit)
+		end
+		log_info(ffi.C.LOG_GRP_PREFILL, "root zone import failed (%s), retry in %s",
+			errmsg, display_delay(rz_cur_interval))
+	else
+		-- re-download before TTL expires
+		rz_cur_interval = (file_ttl - rz_interval_threshold
+					- math.random(rz_interval_randomizer_limit))
+		log_info(ffi.C.LOG_GRP_PREFILL, "root zone refresh in %s",
+			display_delay(rz_cur_interval))
+	end
+	restart_timer(rz_cur_interval)
+end
+
+function prefill.deinit()
+	stop_timer()
+end
+
+-- process one item from configuration table
+-- right now it supports only root zone because
+-- prefill module uses global variables
+local function config_zone(zone_cfg)
+	if zone_cfg.interval then
+		zone_cfg.interval = tonumber(zone_cfg.interval)
+		if zone_cfg.interval < rz_interval_min then
+			error(string.format('[prefil] refresh interval %d s is too short, '
+				.. 'minimal interval is %d s',
+				zone_cfg.interval, rz_interval_min))
+		end
+		rz_default_interval = zone_cfg.interval
+		rz_cur_interval = zone_cfg.interval
+	end
+
+	rz_ca_file = zone_cfg.ca_file
+
+	if not zone_cfg.url or not string.match(zone_cfg.url, '^https://') then
+		error('[prefil] option url must contain a '
+			.. 'https:// URL of a zone file')
+	else
+		rz_url = zone_cfg.url
+	end
+end
+
+function prefill.config(config)
+	if config == nil then return end -- e.g. just modules = { 'prefill' }
+	local root_configured = false
+	if type(config) ~= 'table' then
+		error('[prefil] configuration must be in table '
+			.. '{owner name = {per-zone config}}')
+	end
+	for owner, zone_cfg in pairs(config) do
+		if owner ~= '.' then
+			error('[prefil] only root zone can be imported '
+				.. 'at the moment')
+		else
+			config_zone(zone_cfg)
+			root_configured = true
+		end
+	end
+	if not root_configured then
+		error('[prefil] this module version requires configuration '
+			.. 'for root zone')
+	end
+
+	restart_timer(0)  -- start now
+end
+
+return prefill
diff --git a/modules/prefill/prefill.test/empty.zone b/modules/prefill/prefill.test/empty.zone
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/modules/prefill/prefill.test/empty.zone
diff --git a/modules/prefill/prefill.test/example.com.zone b/modules/prefill/prefill.test/example.com.zone
new file mode 100644
index 0000000..55edbf0
--- /dev/null
+++ b/modules/prefill/prefill.test/example.com.zone
@@ -0,0 +1,12 @@
+; SPDX-License-Identifier: GPL-3.0-or-later
+$ORIGIN example.com.
+$TTL 3600
+
+@	SOA	dns1.example.com. hostmaster.example.com. (
+		2010111213	; serial
+		6h		; refresh
+		1h		; retry
+		1w		; expire
+		1d )		; minimum
+
+	NS	dns1
diff --git a/modules/prefill/prefill.test/prefill.test.lua b/modules/prefill/prefill.test/prefill.test.lua
new file mode 100644
index 0000000..3d39d8f
--- /dev/null
+++ b/modules/prefill/prefill.test/prefill.test.lua
@@ -0,0 +1,123 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+-- unload modules which are not related to this test
+if ta_signal_query then
+        modules.unload('ta_signal_query')
+end
+if priming then
+        modules.unload('priming')
+end
+if detect_time_skew then
+        modules.unload('detect_time_skew')
+end
+
+-- test. domain is used by some tests, allow it
+policy.add(policy.suffix(policy.PASS, {todname('test.')}))
+
+cache.size = 2*MB
+-- log_level('debug')
+
+-- Self-checks on globals
+assert(help() ~= nil)
+assert(worker.id ~= nil)
+-- Self-checks on facilities
+assert(cache.stats() ~= nil)
+assert(cache.backends() ~= nil)
+assert(worker.stats() ~= nil)
+assert(net.interfaces() ~= nil)
+-- Self-checks on loaded stuff
+assert(#modules.list() > 0)
+-- Self-check timers
+ev = event.recurrent(1 * sec, function () return 1 end)
+event.cancel(ev)
+ev = event.after(0, function () return 1 end)
+
+
+-- Import fake root zone; avoid interference with configured keyfile_default.
+trust_anchors.remove('.')
+trust_anchors.add('.			IN DS 18213 7 2 A1D391053583A4BC597DB9588B296060FC55EAC80B3831CA371BA1FA AE997244')
+
+-- do not attempt to contact outside world, operate only on cache
+net.ipv4 = false
+net.ipv6 = false
+-- do not listen, test is driven by config code
+env.KRESD_NO_LISTEN = true
+
+local check_answer = require('test_utils').check_answer
+
+local function zone_import(fname, downgrade)
+	return require('ffi').C.zi_zone_import({
+			zone_file = fname,
+			downgrade = downgrade,
+	})
+end
+
+local function import_valid_root_zone()
+	cache.clear()
+	local import_res = zone_import('testroot.zone')
+	assert(import_res == 0)
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- sanity checks - cache must be filled in
+	ok(cache.count() > 0, 'cache is not empty after import of valid signed root zone')
+	check_answer('root apex is in cache',
+                     '.', kres.type.NS, kres.rcode.NOERROR)
+	check_answer('deep subdomain is in cache',
+		     'a.b.subtree1.', kres.type.AAAA, kres.rcode.NOERROR)
+end
+
+local function import_root_no_soa()
+	cache.clear()
+	local import_res = zone_import('testroot_no_soa.zone')
+	assert(import_res == -1)
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- sanity checks - cache must be filled in
+	ok(cache.count() == 0 , 'cache is still empty after import of zone without SOA record')
+end
+
+local function import_unsigned_root_zone()
+	cache.clear()
+	zone_import('testroot.zone.unsigned')
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- we wanted it to fail
+	ok(cache.count() == 0, 'cache is still empty after import of unsigned zone')
+end
+
+local function import_not_root_zone()
+	cache.clear()
+	zone_import('example.com.zone')
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- we wanted it to fail
+	ok(cache.count() == 0, 'cache is still empty after import of other zone than root')
+end
+
+local function import_empty_zone()
+	cache.clear()
+	local import_res = zone_import('empty.zone')
+	assert(import_res < 0)
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- sanity checks - cache must be filled in
+	ok(cache.count() == 0, 'cache is still empty after import of empty zone')
+end
+
+local function import_random_trash()
+	cache.clear()
+	local import_res = zone_import('random.zone')
+	assert(import_res < 0)
+	-- beware that import takes at least 100 ms
+	worker.sleep(0.2)  -- zimport is delayed by 100 ms from function call
+	-- sanity checks - cache must be filled in
+	ok(cache.count() == 0, 'cache is still empty after import of unparseable file')
+end
+
+return {
+	import_valid_root_zone,
+	import_root_no_soa,
+	import_unsigned_root_zone,
+	import_not_root_zone,
+	import_empty_zone,
+	import_random_trash,
+}
diff --git a/modules/prefill/prefill.test/random.zone b/modules/prefill/prefill.test/random.zone
new file mode 100644
index 0000000..1650ac2
--- /dev/null
+++ b/modules/prefill/prefill.test/random.zone
@@ -0,0 +1,2 @@
+4����=g<����k~����>B�iڴ=�FN5�0H.��Áa�@汁wU�ټ��Pn�2ޗt}��3�q�U����l�ΤQIa
�y�G�D#�	�ֈ>���Sdj��U?ʨ�T����WMC}2 )�`a����	*lj7V�5�%圅.�	���eZ5BI���Sޚ���Lv>��|��<dF�;�6�	��G��L{tɹ�݌*C��cj$�G)���I��C�0�}�t���NK��^��d����
+; SPDX-License-Identifier: GPL-3.0-or-later
diff --git a/modules/prefill/prefill.test/testroot.zone b/modules/prefill/prefill.test/testroot.zone
new file mode 100644
index 0000000..76b9d18
--- /dev/null
+++ b/modules/prefill/prefill.test/testroot.zone
@@ -0,0 +1,59 @@
+; SPDX-License-Identifier: GPL-3.0-or-later
+; File written on Thu Aug 22 15:47:02 2019
+; dnssec_signzone version 9.11.3-1ubuntu1.8-Ubuntu
+.			86400	IN SOA	rootns. you.test. (
+					2017071102 ; serial
+					1800       ; refresh (30 minutes)
+					900        ; retry (15 minutes)
+					604800     ; expire (1 week)
+					86400      ; minimum (1 day)
+					)
+			86400	RRSIG	SOA 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					QRmlVBvwhkNqmsp4H9zxcPeVg++5g/eR8EPb
+					DldazjUIoqbQYarTD+3DDf8tvwJu1aBNvBhm
+					cQCauTS5JWzisg== )
+			86400	NS	rootns.
+			86400	RRSIG	NS 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					kMiL+id0WrESTSw51qI96kbolLTegn+Uraim
+					8GjNr0d2fH8m885ORkr7C4g0RrzfAKNokArF
+					rQltwL8sMowgJg== )
+			86400	NSEC	a.b.subtree1. NS SOA RRSIG NSEC DNSKEY
+			86400	RRSIG	NSEC 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					zv+f8FELPfYeWn7Ryy/+rBR+qASu4QC6gAka
+					vpeWRgybUQh50LrJIxINuf0YLCpqxjsX6zkK
+					zwbt0BgPHjfRHA== )
+			86400	DNSKEY	256 3 7 (
+					AwEAAc9BtlREycexYH5az+dIbhI6sM56F+kd
+					SI43ZTGNT/Bam5vGrXju0uTHCJ2+KBwOSz7d
+					ZVchX0ulIJOUV9MteT0=
+					) ; ZSK; alg = NSEC3RSASHA1 ; key id = 46349
+			86400	DNSKEY	257 3 7 (
+					AwEAAcEFKHPyE1JMfRLhJK9mgcBZ+TR0Pj6u
+					shF6YbLkQoRs6Uzm458ErCcAdukJsTckqCzq
+					PFEqGLRztzyAJ7Z3G0k=
+					) ; KSK; alg = NSEC3RSASHA1 ; key id = 18213
+			86400	RRSIG	DNSKEY 7 0 86400 (
+					20500101000000 20190822124702 18213 .
+					ih4ScNqt9muT/Dqc05oO5T/xAyRK1/LblHph
+					GHedPHW7mC6IzsDBbqjD/P1nVK5RkM2Q+ozV
+					Ltbtmt2CafXsLA== )
+			86400	RRSIG	DNSKEY 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					Lqle63cGoJdZA4CHHUq3ZqFxsbYATelzj5Dl
+					lDcc7vLbn2Qy9AVUC5I6UdZqMK2UDyO+DWCG
+					Cmq5705eAQlcsQ== )
+a.b.subtree1.		86400	IN AAAA	2001:db8::
+			86400	RRSIG	AAAA 7 3 86400 (
+					20500101000000 20190822124702 46349 .
+					Hq6CUu3CVN/90b1Sozv0uIgH5ePxY3olc3eq
+					PoeyfdS+3HjSgb+Ji+GjYAAOMaVDS0APwwMe
+					pHxhdgO/zpKHRQ== )
+			86400	NSEC	. AAAA RRSIG NSEC
+			86400	RRSIG	NSEC 7 3 86400 (
+					20500101000000 20190822124702 46349 .
+					lcJ7xdzxgTTvj2JiwzhDRyxTx2ZJ5zwzx0hC
+					ttTrSfG+2GyMnPzJ9MFid5S2w0WbWOWWLaKH
+					O0ucI8xvYInNAA== )
diff --git a/modules/prefill/prefill.test/testroot.zone.unsigned b/modules/prefill/prefill.test/testroot.zone.unsigned
new file mode 100644
index 0000000..fbc551f
--- /dev/null
+++ b/modules/prefill/prefill.test/testroot.zone.unsigned
@@ -0,0 +1,4 @@
+; SPDX-License-Identifier: GPL-3.0-or-later
+.		86400	SOA	rootns. you.test. 2017071101 1800 900 604800 86400
+.		86400	NS	rootns.
+a.b.subtree1.	86400	AAAA	2001:db8::
diff --git a/modules/prefill/prefill.test/testroot_no_soa.zone b/modules/prefill/prefill.test/testroot_no_soa.zone
new file mode 100644
index 0000000..e3ad08c
--- /dev/null
+++ b/modules/prefill/prefill.test/testroot_no_soa.zone
@@ -0,0 +1,52 @@
+; SPDX-License-Identifier: GPL-3.0-or-later
+; File written on Thu Aug 22 15:47:02 2019
+; dnssec_signzone version 9.11.3-1ubuntu1.8-Ubuntu
+.			86400	RRSIG	SOA 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					QRmlVBvwhkNqmsp4H9zxcPeVg++5g/eR8EPb
+					DldazjUIoqbQYarTD+3DDf8tvwJu1aBNvBhm
+					cQCauTS5JWzisg== )
+			86400	NS	rootns.
+			86400	RRSIG	NS 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					kMiL+id0WrESTSw51qI96kbolLTegn+Uraim
+					8GjNr0d2fH8m885ORkr7C4g0RrzfAKNokArF
+					rQltwL8sMowgJg== )
+			86400	NSEC	a.b.subtree1. NS SOA RRSIG NSEC DNSKEY
+			86400	RRSIG	NSEC 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					zv+f8FELPfYeWn7Ryy/+rBR+qASu4QC6gAka
+					vpeWRgybUQh50LrJIxINuf0YLCpqxjsX6zkK
+					zwbt0BgPHjfRHA== )
+			86400	DNSKEY	256 3 7 (
+					AwEAAc9BtlREycexYH5az+dIbhI6sM56F+kd
+					SI43ZTGNT/Bam5vGrXju0uTHCJ2+KBwOSz7d
+					ZVchX0ulIJOUV9MteT0=
+					) ; ZSK; alg = NSEC3RSASHA1 ; key id = 46349
+			86400	DNSKEY	257 3 7 (
+					AwEAAcEFKHPyE1JMfRLhJK9mgcBZ+TR0Pj6u
+					shF6YbLkQoRs6Uzm458ErCcAdukJsTckqCzq
+					PFEqGLRztzyAJ7Z3G0k=
+					) ; KSK; alg = NSEC3RSASHA1 ; key id = 18213
+			86400	RRSIG	DNSKEY 7 0 86400 (
+					20500101000000 20190822124702 18213 .
+					ih4ScNqt9muT/Dqc05oO5T/xAyRK1/LblHph
+					GHedPHW7mC6IzsDBbqjD/P1nVK5RkM2Q+ozV
+					Ltbtmt2CafXsLA== )
+			86400	RRSIG	DNSKEY 7 0 86400 (
+					20500101000000 20190822124702 46349 .
+					Lqle63cGoJdZA4CHHUq3ZqFxsbYATelzj5Dl
+					lDcc7vLbn2Qy9AVUC5I6UdZqMK2UDyO+DWCG
+					Cmq5705eAQlcsQ== )
+a.b.subtree1.		86400	IN AAAA	2001:db8::
+			86400	RRSIG	AAAA 7 3 86400 (
+					20500101000000 20190822124702 46349 .
+					Hq6CUu3CVN/90b1Sozv0uIgH5ePxY3olc3eq
+					PoeyfdS+3HjSgb+Ji+GjYAAOMaVDS0APwwMe
+					pHxhdgO/zpKHRQ== )
+			86400	NSEC	. AAAA RRSIG NSEC
+			86400	RRSIG	NSEC 7 3 86400 (
+					20500101000000 20190822124702 46349 .
+					lcJ7xdzxgTTvj2JiwzhDRyxTx2ZJ5zwzx0hC
+					ttTrSfG+2GyMnPzJ9MFid5S2w0WbWOWWLaKH
+					O0ucI8xvYInNAA== )