Adding upstream version 5.7.1.upstream/5.7.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 418 insertions, 0 deletions

diff --git a/modules/http/http.lua.in b/modules/http/http.lua.in
new file mode 100644
index 0000000..b6cf16a
--- /dev/null
+++ b/modules/http/http.lua.in
@@ -0,0 +1,418 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+
+-- Load dependent modules
+if not stats then modules.load('stats') end
+if not bogus_log then modules.load('bogus_log') end
+
+local ffi = require('ffi')
+local cqueues = require('cqueues')
+cqueues.socket = require('cqueues.socket')
+assert(cqueues.VERSION >= 20150112)  -- fdopen changed semantics
+
+-- This is a module that does the heavy lifting to provide an HTTP/2 enabled
+-- server that supports TLS by default and provides endpoint for other modules
+-- in order to enable them to export restful APIs and websocket streams.
+-- One example is statistics module that can stream live metrics on the website,
+-- or publish metrics on request for Prometheus scraper.
+local http_server = require('http.server')
+local http_headers = require('http.headers')
+local http_websocket = require('http.websocket')
+local http_util = require "http.util"
+local has_mmdb, mmdb  = pcall(require, 'mmdb')
+
+-- A sub-module for certificate management.
+local tls_cert = require('kres_modules.http_tls_cert')
+
+-- Module declaration
+local M = {
+	servers = {},
+	configs = { _builtin = {} }  -- configuration templates
+}
+
+-- inherited by all configurations
+M.configs._builtin._all = {
+	cq = worker.bg_worker.cq,
+	cert = 'self.crt',
+	key = 'self.key',
+	ephemeral = true,
+	client_timeout = 5
+}
+-- log errors but do not throw
+M.configs._builtin._all.onerror = function(myserver, context, op, err, errno) -- luacheck: ignore 212
+	local msg = op .. ' on ' .. tostring(context) .. ' failed'
+	if err then
+		msg = msg .. ': ' .. tostring(err)
+	end
+	log_info(ffi.C.LOG_GRP_HTTP, msg)
+end
+
+-- M.config() without explicit "kind" modifies this
+M.configs._all = {}
+
+-- DoH
+M.configs._builtin.doh_legacy = {}
+
+-- management endpoint
+M.configs._builtin.webmgmt = {}
+
+-- Map extensions to MIME type
+local mime_types = {
+	js = 'application/javascript',
+	css = 'text/css',
+	tpl = 'text/html',
+	ico = 'image/x-icon'
+}
+
+-- Preload static contents, nothing on runtime will touch the disk
+local function pgload(relpath, modname)
+	if not modname then modname = 'http' end
+	local fp, err = io.open(string.format(
+        '@modules_dir@/%s/%s', modname, relpath), 'r')
+	if not fp then
+		fp, err = io.open(string.format(
+            '@modules_dir@/%s/static/%s', modname, relpath), 'r')
+	end
+	if not fp then error(err) end
+	local data = fp:read('*all')
+	fp:close()
+	-- Guess content type
+	local ext = relpath:match('[^\\.]+$')
+	return {mime_types[ext] or 'text', data, nil, 86400}
+end
+M.page = pgload
+
+-- Preloaded static assets
+local pages = {
+	'favicon.ico',
+	'kresd.js',
+	'kresd.css',
+	'jquery.js',
+	'd3.js',
+	'topojson.js',
+	'datamaps.world.min.js',
+	'dygraph.min.js',
+	'selectize.min.js',
+	'selectize.bootstrap3.css',
+	'bootstrap.min.js',
+	'bootstrap.min.css',
+	'bootstrap-theme.min.css',
+	'glyphicons-halflings-regular.woff2',
+}
+
+-- Serve preloaded root page
+local function serve_root()
+	local data = pgload('main.tpl')[2]
+	data = data
+	        :gsub('{{ title }}', M.title or ('kresd @ ' .. hostname()))
+	        :gsub('{{ host }}', hostname())
+	return function (_, _)
+		-- Render snippets
+		local rsnippets = {}
+		for _,v in pairs(M.snippets) do
+			local sid = string.lower(string.gsub(v[1], ' ', '-'))
+			table.insert(rsnippets, string.format('<section id="%s"><h2>%s</h2>\n%s</section>', sid, v[1], v[2]))
+		end
+		-- Return index page
+		return data
+		        :gsub('{{ snippets }}', table.concat(rsnippets, '\n'))
+	end
+end
+
+-- Export HTTP service endpoints
+M.configs._builtin.doh_legacy.endpoints = {}
+M.configs._builtin.webmgmt.endpoints = {}
+local mgmt_endpoints = M.configs._builtin.webmgmt.endpoints
+
+mgmt_endpoints['/'] = {'text/html', serve_root()}
+
+-- Export static pages
+for _, pg in ipairs(pages) do
+	mgmt_endpoints['/'..pg] = pgload(pg)
+end
+
+-- Export built-in prometheus interface
+local prometheus = require('kres_modules.prometheus')
+for k, v in pairs(prometheus.endpoints) do
+	mgmt_endpoints[k] = v
+end
+M.prometheus = prometheus
+
+-- Export built-in trace interface
+local http_trace = require('kres_modules.http_trace')
+for k, v in pairs(http_trace.endpoints) do
+	mgmt_endpoints[k] = v
+end
+M.trace = http_trace
+
+M.configs._builtin.doh_legacy.endpoints = {}
+local http_doh = require('kres_modules.http_doh')
+for k, v in pairs(http_doh.endpoints) do
+	mgmt_endpoints[k] = v
+	M.configs._builtin.doh_legacy.endpoints[k] = v
+end
+M.doh = http_doh
+
+-- Export HTTP service page snippets
+M.snippets = {}
+
+-- Serve known requests, for methods other than GET
+-- the endpoint must be a closure and not a preloaded string
+local function serve(endpoints, h, stream)
+	local hsend = http_headers.new()
+	local path = h:get(':path')
+	local entry = endpoints[path]
+	if not entry then -- Accept top-level path match
+		entry = endpoints[path:match '^/[^/?]*']
+	end
+	-- Unpack MIME and data
+	local data, mime, ttl, any_origin, err
+	if entry then
+		mime = entry[1]
+		data = entry[2]
+		ttl = entry[4]
+		any_origin = entry[5]
+	end
+	-- Get string data out of service endpoint
+	if type(data) == 'function' then
+		local set_mime, set_ttl
+		data, err, set_mime, set_ttl = data(h, stream)
+		-- Override default endpoint mime/TTL
+		if set_mime then mime = set_mime end
+		if set_ttl then ttl = set_ttl end
+		-- Handler doesn't provide any data
+		if data == false then return end
+		if type(data) == 'number' then return tostring(data), err end
+	-- Methods other than GET require handler to be closure
+	elseif h:get(':method') ~= 'GET' then
+		return '501', ''
+	end
+	if type(data) == 'table' then data = tojson(data) end
+	if not mime or type(data) ~= 'string' then
+		return '404', ''
+	else
+		-- Serve content type appropriately
+		hsend:append(':status', '200')
+		hsend:append('content-type', mime)
+		hsend:append('content-length', tostring(#data))
+		if ttl then
+			hsend:append('cache-control', string.format('max-age=%d', ttl))
+		end
+		if any_origin then
+			hsend:append('access-control-allow-origin', '*')
+		end
+		assert(stream:write_headers(hsend, false))
+		assert(stream:write_chunk(data, true))
+	end
+end
+
+-- Web server service closure
+local function route(endpoints)
+	assert(type(endpoints) == 'table', 'endpoints are not a table, is it a botched template?')
+	return function (_, stream)
+		-- HTTP/2: We're only permitted to send in open/half-closed (remote)
+		local connection = stream.connection
+		if connection.version >= 2 then
+			if stream.state ~= 'open' and stream.state ~= 'half closed (remote)' then
+				return
+			end
+		end
+		-- Start reading headers
+		local h = assert(stream:get_headers())
+		local m = h:get(':method')
+		local path = h:get(':path')
+
+		-- Upgrade connection to WebSocket
+		local ws = http_websocket.new_from_stream(stream, h)
+		if ws then
+			log_info(ffi.C.LOG_GRP_HTTP, '%s %s HTTP/%s web socket open',
+				m, path, tostring(connection.version))
+			assert(ws:accept { protocols = {'json'} })
+			-- Continue streaming results to client
+			local ep = endpoints[path]
+			local cb = ep[3]
+			if cb then
+				cb(h, ws)
+			end
+			ws:close()
+			log_info(ffi.C.LOG_GRP_HTTP, '%s %s HTTP/%s web socket closed',
+				m, path, tostring(connection.version))
+			return
+		else
+			local ok, err, reason = http_util.yieldable_pcall(serve, endpoints, h, stream)
+			if not ok or err then
+				err = err or '500'
+				log_info(ffi.C.LOG_GRP_HTTP, '%s %s HTTP/%s %s %s',
+					m, path, tostring(connection.version), err, reason or '')
+				-- Method is not supported
+				local hsend = http_headers.new()
+				hsend:append(':status', err)
+				if reason then
+					assert(stream:write_headers(hsend, false))
+					assert(stream:write_chunk(reason, true))
+				else
+					assert(stream:write_headers(hsend, true))
+				end
+			else
+				log_info(ffi.C.LOG_GRP_HTTP, '%s %s HTTP/%s 200',
+					m, path, tostring(connection.version))
+			end
+
+		end
+	end
+end
+
+-- @function Merge dictionaries, nil is like empty dict.
+-- Values from right-hand side dictionaries take precedence.
+local function mergeconf(...)
+	local merged = {}
+	local ntables = select('#', ...)
+	local tables = {...}
+	for i = 1, ntables do
+		local intable = tables[i]
+		if intable ~= nil then
+			assert(type(intable) == 'table', 'cannot merge non-tables')
+			for key, val in pairs(intable) do
+				merged[key] = val
+			end
+		end
+	end
+	return merged
+end
+
+-- @function Listen on given socket
+-- using configuration for specific "kind" of HTTP server
+local function add_socket(fd, kind, addr_str)
+	assert(M.servers[fd] == nil, 'socket is already served by an HTTP instance')
+	local conf = mergeconf(M.configs._builtin._all, M.configs._builtin[kind],
+						M.configs._all, M.configs[kind])
+	conf.socket = cqueues.socket.fdopen({ fd = fd, reuseport = true, reuseaddr = true })
+	if conf.tls ~= false then -- Create a TLS context, either from files or new.
+		if conf.ephemeral then
+			if not M.ephem_state then
+				M.ephem_state = { servers = M.servers }
+				tls_cert.ephemeral_state_maintain(M.ephem_state, conf.cert, conf.key)
+			end
+			conf.ctx = M.ephem_state.ctx
+		else
+			local certs, key = tls_cert.load(conf.cert, conf.key)
+			conf.ctx = tls_cert.new_tls_context(certs, key)
+		end
+		assert(conf.ctx)
+	end
+	-- Compose server handler
+	local routes = route(conf.endpoints)
+	conf.onstream = routes
+	-- Create TLS context and start listening
+	local s, err = http_server.new(conf)
+	-- Manually call :listen() so that we are bound before calling :localname()
+	if s then
+		err = select(2, s:listen())
+	end
+	if err then
+		panic('failed to listen on %s: %s', addr_str, err)
+	end
+	M.servers[fd] = {kind = kind, server = s, config = conf}
+end
+
+-- @function Stop listening on given socket
+local function remove_socket(fd)
+	local instance = M.servers[fd]
+	assert(instance, 'HTTP module is not listening on given socket')
+
+	instance.server:close()
+	M.servers[fd] = nil
+end
+
+-- @function Listen for config changes from net.listen()/net.close()
+local function cb_socket(...)
+	local added, endpoint, addr_str = unpack({...})
+	endpoint = ffi.cast('struct endpoint **', endpoint)[0]
+	local kind = ffi.string(endpoint.flags.kind)
+	local socket = endpoint.fd
+	if added then
+		return add_socket(socket, kind, addr_str)
+	else
+		return remove_socket(socket)
+	end
+end
+
+-- @function Init module
+function M.init()
+	net.register_endpoint_kind('doh_legacy', cb_socket)
+	net.register_endpoint_kind('webmgmt', cb_socket)
+end
+
+-- @function Cleanup module
+function M.deinit()
+	for fd, _ in pairs(M.servers) do
+		remove_socket(fd)
+	end
+	tls_cert.ephemeral_state_destroy(M.ephem_state)
+	net.register_endpoint_kind('doh_legacy')
+	net.register_endpoint_kind('webmgmt')
+end
+
+-- @function Configure module, i.e. store new configuration template
+-- kind = socket type (doh_legacy/webmgmt)
+function M.config(conf, kind)
+	if conf == nil and kind == nil then
+		-- default module config, nothing to do
+		return
+	end
+
+	kind = kind or '_all'
+	assert(type(kind) == 'string')
+
+	local operation
+	-- builtins cannot be removed or added
+	if M.configs._builtin[kind] then
+		operation = 'modify'
+		conf = conf or {}
+	elseif M.configs[kind] then  -- config on an existing user template
+		if conf then	operation = 'modify'
+		else		operation = 'delete' end
+	else  -- config for not-yet-existing template
+		if conf then	operation = 'add'
+		else		panic('[http] endpoint kind "%s" does not exist, '
+					.. 'nothing to delete', kind) end
+	end
+
+	if operation == 'modify' or operation == 'add' then
+		assert(type(conf) == 'table', 'config { cert = "...", key = "..." }')
+
+		if conf.cert then
+			conf.ephemeral = false
+			if not conf.key then
+				panic('[http] certificate provided, but missing key')
+			end
+			-- test if it can be loaded or not
+			tls_cert.load(conf.cert, conf.key)
+		end
+		if conf.geoip then
+			if has_mmdb then
+				M.geoip = mmdb.open(conf.geoip)
+			else
+				error('[http] mmdblua library not found, please remove GeoIP configuration')
+			end
+		end
+	end
+
+	for _, instance in pairs(M.servers) do
+		-- modification cannot be implemented as
+		-- remove_socket + add_socket because remove closes the socket
+		if instance.kind == kind or kind == '_all' then
+			panic('unable to modify configuration for '
+				.. 'endpoint kind "%s" because it is in '
+				.. 'use, use net.close() first', kind)
+		end
+	end
+
+	if operation == 'add' then
+		net.register_endpoint_kind(kind, cb_socket)
+	elseif operation == 'delete' then
+		net.register_endpoint_kind(kind)
+	end
+	M.configs[kind] = conf
+end
+
+return M