diff options
Diffstat (limited to '')
| -rw-r--r-- | src/libknot/xdp.h | 35 | ||||
| -rw-r--r-- | src/libknot/xdp/Makefile.am | 20 | ||||
| -rw-r--r-- | src/libknot/xdp/Makefile.in | 545 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-consts.h | 57 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel-obj.c | 941 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel-obj.h | 2 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel.c | 293 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-user.c | 313 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-user.h | 139 | ||||
| -rw-r--r-- | src/libknot/xdp/eth.c | 312 | ||||
| -rw-r--r-- | src/libknot/xdp/eth.h | 111 | ||||
| -rw-r--r-- | src/libknot/xdp/msg.h | 62 | ||||
| -rw-r--r-- | src/libknot/xdp/msg_init.h | 74 | ||||
| -rw-r--r-- | src/libknot/xdp/protocols.h | 446 | ||||
| -rw-r--r-- | src/libknot/xdp/quic.c | 1028 | ||||
| -rw-r--r-- | src/libknot/xdp/quic.h | 134 | ||||
| -rw-r--r-- | src/libknot/xdp/quic_conn.c | 506 | ||||
| -rw-r--r-- | src/libknot/xdp/quic_conn.h | 314 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp.c | 729 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp.h | 227 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp_iobuf.c | 266 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp_iobuf.h | 120 | ||||
| -rw-r--r-- | src/libknot/xdp/xdp.c | 568 | ||||
| -rw-r--r-- | src/libknot/xdp/xdp.h | 189 |
24 files changed, 7431 insertions, 0 deletions
diff --git a/src/libknot/xdp.h b/src/libknot/xdp.h new file mode 100644 index 0000000..db4460c --- /dev/null +++ b/src/libknot/xdp.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2021 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief Convenience header for including XDP-related stuff. + * + * \addtogroup libknot + * @{ + */ + +#pragma once + +#if ENABLE_XDP +#include "libknot/xdp/xdp.h" +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/eth.h" +#include "libknot/xdp/tcp.h" +#endif + +/*! @} */ diff --git a/src/libknot/xdp/Makefile.am b/src/libknot/xdp/Makefile.am new file mode 100644 index 0000000..cb58438 --- /dev/null +++ b/src/libknot/xdp/Makefile.am @@ -0,0 +1,20 @@ +# Useful commands: +# make filter +# ip link show $eth +# sudo ip link set dev $eth xdp off +# sudo ip link set dev $eth xdp obj ./bpf-kernel.o +# +# Built using LLVM/CLANG 14. +# +# When updating check using `llvm-objdump -h bpf-kernel.o` if .BTF and .BTF.ext +# sections are present. + +EXTRA_DIST = bpf-kernel-obj.c bpf-kernel.c + +.PHONY: filter + +filter: + rm -f bpf-kernel.o bpf-kernel-obj.c + clang -target bpf -Wall -O2 -g -DNDEBUG -c -o bpf-kernel.o -I/usr/include/x86_64-linux-gnu -include ../../config.h bpf-kernel.c + llvm-strip -S bpf-kernel.o + xxd -i bpf-kernel.o > bpf-kernel-obj.c diff --git a/src/libknot/xdp/Makefile.in b/src/libknot/xdp/Makefile.in new file mode 100644 index 0000000..b56c39d --- /dev/null +++ b/src/libknot/xdp/Makefile.in @@ -0,0 +1,545 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +# Useful commands: +# make filter +# ip link show $eth +# sudo ip link set dev $eth xdp off +# sudo ip link set dev $eth xdp obj ./bpf-kernel.o +# +# Built using LLVM/CLANG 14. +# +# When updating check using `llvm-objdump -h bpf-kernel.o` if .BTF and .BTF.ext +# sections are present. +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/libknot/xdp +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_check_link_flag.m4 \ + $(top_srcdir)/m4/code-coverage.m4 \ + $(top_srcdir)/m4/knot-lib-version.m4 \ + $(top_srcdir)/m4/knot-module.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/sanitizer.m4 $(top_srcdir)/m4/visibility.m4 \ + $(top_srcdir)/m4/knot-version.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/src/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +SOURCES = +DIST_SOURCES = +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +am__DIST_COMMON = $(srcdir)/Makefile.in +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CFLAG_VISIBILITY = @CFLAG_VISIBILITY@ +CODE_COVERAGE_ENABLED = @CODE_COVERAGE_ENABLED@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DNSTAP_CFLAGS = @DNSTAP_CFLAGS@ +DNSTAP_LIBS = @DNSTAP_LIBS@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GENHTML = @GENHTML@ +GREP = @GREP@ +HAVE_VISIBILITY = @HAVE_VISIBILITY@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KNOT_VERSION_MAJOR = @KNOT_VERSION_MAJOR@ +KNOT_VERSION_MINOR = @KNOT_VERSION_MINOR@ +KNOT_VERSION_PATCH = @KNOT_VERSION_PATCH@ +LCOV = @LCOV@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LDFLAG_EXCLUDE_LIBS = @LDFLAG_EXCLUDE_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +LT_NO_UNDEFINED = @LT_NO_UNDEFINED@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PDFLATEX = @PDFLATEX@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +PROTOC_C = @PROTOC_C@ +RANLIB = @RANLIB@ +RELEASE_DATE = @RELEASE_DATE@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SPHINXBUILD = @SPHINXBUILD@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +cap_ng_CFLAGS = @cap_ng_CFLAGS@ +cap_ng_LIBS = @cap_ng_LIBS@ +conf_mapsize = @conf_mapsize@ +config_dir = @config_dir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dlopen_LIBS = @dlopen_LIBS@ +docdir = @docdir@ +dvidir = @dvidir@ +embedded_libbpf_CFLAGS = @embedded_libbpf_CFLAGS@ +embedded_libbpf_LIBS = @embedded_libbpf_LIBS@ +embedded_libngtcp2_CFLAGS = @embedded_libngtcp2_CFLAGS@ +embedded_libngtcp2_LIBS = @embedded_libngtcp2_LIBS@ +exec_prefix = @exec_prefix@ +fuzzer_CFLAGS = @fuzzer_CFLAGS@ +fuzzer_LDFLAGS = @fuzzer_LDFLAGS@ +gnutls_CFLAGS = @gnutls_CFLAGS@ +gnutls_LIBS = @gnutls_LIBS@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libbpf_CFLAGS = @libbpf_CFLAGS@ +libbpf_LIBS = @libbpf_LIBS@ +libdir = @libdir@ +libdnssec_SONAME = @libdnssec_SONAME@ +libdnssec_SOVERSION = @libdnssec_SOVERSION@ +libdnssec_VERSION_INFO = @libdnssec_VERSION_INFO@ +libedit_CFLAGS = @libedit_CFLAGS@ +libedit_LIBS = @libedit_LIBS@ +libelf_CFLAGS = @libelf_CFLAGS@ +libelf_LIBS = @libelf_LIBS@ +libexecdir = @libexecdir@ +libfstrm_CFLAGS = @libfstrm_CFLAGS@ +libfstrm_LIBS = @libfstrm_LIBS@ +libidn2_CFLAGS = @libidn2_CFLAGS@ +libidn2_LIBS = @libidn2_LIBS@ +libidn_CFLAGS = @libidn_CFLAGS@ +libidn_LIBS = @libidn_LIBS@ +libknot_SONAME = @libknot_SONAME@ +libknot_SOVERSION = @libknot_SOVERSION@ +libknot_VERSION_INFO = @libknot_VERSION_INFO@ +libkqueue_CFLAGS = @libkqueue_CFLAGS@ +libkqueue_LIBS = @libkqueue_LIBS@ +libmaxminddb_CFLAGS = @libmaxminddb_CFLAGS@ +libmaxminddb_LIBS = @libmaxminddb_LIBS@ +libmnl_CFLAGS = @libmnl_CFLAGS@ +libmnl_LIBS = @libmnl_LIBS@ +libnghttp2_CFLAGS = @libnghttp2_CFLAGS@ +libnghttp2_LIBS = @libnghttp2_LIBS@ +libngtcp2_CFLAGS = @libngtcp2_CFLAGS@ +libngtcp2_LIBS = @libngtcp2_LIBS@ +libprotobuf_c_CFLAGS = @libprotobuf_c_CFLAGS@ +libprotobuf_c_LIBS = @libprotobuf_c_LIBS@ +liburcu_CFLAGS = @liburcu_CFLAGS@ +liburcu_LIBS = @liburcu_LIBS@ +liburcu_PKGCONFIG = @liburcu_PKGCONFIG@ +libxdp_CFLAGS = @libxdp_CFLAGS@ +libxdp_LIBS = @libxdp_LIBS@ +libzscanner_SONAME = @libzscanner_SONAME@ +libzscanner_SOVERSION = @libzscanner_SOVERSION@ +libzscanner_VERSION_INFO = @libzscanner_VERSION_INFO@ +lmdb_CFLAGS = @lmdb_CFLAGS@ +lmdb_LIBS = @lmdb_LIBS@ +localedir = @localedir@ +localstatedir = @localstatedir@ +malloc_LIBS = @malloc_LIBS@ +mandir = @mandir@ +math_LIBS = @math_LIBS@ +mkdir_p = @mkdir_p@ +module_dir = @module_dir@ +module_instdir = @module_instdir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgconfigdir = @pkgconfigdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pthread_LIBS = @pthread_LIBS@ +run_dir = @run_dir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +storage_dir = @storage_dir@ +sysconfdir = @sysconfdir@ +systemd_CFLAGS = @systemd_CFLAGS@ +systemd_LIBS = @systemd_LIBS@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +EXTRA_DIST = bpf-kernel-obj.c bpf-kernel.c +all: all-am + +.SUFFIXES: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/libknot/xdp/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/libknot/xdp/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +tags TAGS: + +ctags CTAGS: + +cscope cscopelist: + + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool mostlyclean-am + +distclean: distclean-am + -rm -f Makefile +distclean-am: clean-am distclean-generic + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-generic mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: all all-am check check-am clean clean-generic clean-libtool \ + cscopelist-am ctags-am distclean distclean-generic \ + distclean-libtool distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +.PHONY: filter + +filter: + rm -f bpf-kernel.o bpf-kernel-obj.c + clang -target bpf -Wall -O2 -g -DNDEBUG -c -o bpf-kernel.o -I/usr/include/x86_64-linux-gnu -include ../../config.h bpf-kernel.c + llvm-strip -S bpf-kernel.o + xxd -i bpf-kernel.o > bpf-kernel-obj.c + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/libknot/xdp/bpf-consts.h b/src/libknot/xdp/bpf-consts.h new file mode 100644 index 0000000..3b92cbb --- /dev/null +++ b/src/libknot/xdp/bpf-consts.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP filter configuration constants. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <linux/types.h> + +#define KNOT_XDP_PKT_ALIGNMENT 2 /*!< Fix for misaligned access to packet structures. */ + +/*! \brief XDP filter configuration flags. */ +typedef enum { + KNOT_XDP_FILTER_ON = 1 << 0, /*!< Filter enabled. */ + KNOT_XDP_FILTER_UDP = 1 << 1, /*!< Apply filter to UDP. */ + KNOT_XDP_FILTER_TCP = 1 << 2, /*!< Apply filter to TCP. */ + KNOT_XDP_FILTER_QUIC = 1 << 3, /*!< Apply filter to QUIC/UDP. */ + KNOT_XDP_FILTER_PASS = 1 << 4, /*!< Pass incoming messages to ports >= port value. */ + KNOT_XDP_FILTER_DROP = 1 << 5, /*!< Drop incoming messages to ports >= port value. */ + KNOT_XDP_FILTER_ROUTE = 1 << 6, /*!< Consider routing information from kernel. */ +} knot_xdp_filter_flag_t; + +/*! \brief XDP map item for the filter configuration. */ +typedef struct knot_xdp_opts knot_xdp_opts_t; +struct knot_xdp_opts { + __u16 flags; /*!< XDP filter flags \a knot_xdp_filter_flag_t. */ + __u16 udp_port; /*!< UDP/TCP port to listen on. */ + __u16 quic_port; /*!< QUIC/UDP port to listen on. */ +} __attribute__((packed)); + +/*! \brief Additional information from the filter. */ +typedef struct knot_xdp_info knot_xdp_info_t; +struct knot_xdp_info { + __u16 out_if_index; /*!< Index of the output interface (if routing enabled). */ +}; + +/*! @} */ diff --git a/src/libknot/xdp/bpf-kernel-obj.c b/src/libknot/xdp/bpf-kernel-obj.c new file mode 100644 index 0000000..9f1ee71 --- /dev/null +++ b/src/libknot/xdp/bpf-kernel-obj.c @@ -0,0 +1,941 @@ +unsigned char bpf_kernel_o[] = { + 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x0d, 0x00, 0x01, 0x00, 0xbf, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x61, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x1a, 0xfc, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x02, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, 0x18, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x71, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x09, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x29, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x15, 0x02, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xa0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0x98, 0xff, 0x00, 0x00, 0x00, 0x00, 0x71, 0x18, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x11, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x1a, 0xa8, 0xff, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, + 0x36, 0x00, 0x00, 0x00, 0x61, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x61, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x72, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x3a, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x62, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x21, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x74, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x73, 0x0d, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x4f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0b, 0x00, + 0x81, 0x00, 0x00, 0x00, 0xbf, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x01, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x21, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xb0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x74, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x73, 0x11, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x4f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x15, 0x03, 0x1e, 0x00, 0x86, 0xdd, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x55, 0x03, 0xdc, 0x00, + 0x08, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x34, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x04, 0xd3, 0x00, + 0x40, 0x00, 0x00, 0x00, 0xbf, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x15, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6d, 0x45, 0xcd, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x15, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x05, 0x00, 0x00, 0xbf, 0xff, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x4a, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x05, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x3c, 0x00, 0x00, 0x00, 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x10, 0x09, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x05, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x2d, 0x25, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x03, 0xb8, 0x00, + 0x60, 0x00, 0x00, 0x00, 0xbf, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x14, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x04, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x07, 0x04, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x34, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7b, 0x3a, 0x90, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x10, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x08, 0x00, + 0x2c, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x05, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x2d, 0x25, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x71, 0x10, 0x28, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x7b, 0x3a, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, 0x67, 0x08, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xa8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x4f, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x17, 0x00, 0x11, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x55, 0x03, 0x9d, 0x00, + 0x06, 0x00, 0x00, 0x00, 0xbf, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x52, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x02, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xbf, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x1d, 0x32, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x8c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x08, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x2d, 0x28, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x28, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x53, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6d, 0x23, 0x7f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x52, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x02, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x83, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x1d, 0x32, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x08, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x3d, 0x82, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xa0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x79, 0xa5, 0x98, 0xff, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x53, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x03, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x1d, 0x32, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x03, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x15, 0x03, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x05, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x2d, 0x25, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x55, 0x02, 0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, + 0xff, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x55, 0x04, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x09, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x15, 0x09, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xf0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xe8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xe0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xd0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc8, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x63, 0x2a, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa2, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x02, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x73, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x2a, 0xc8, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x11, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x1a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x73, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x61, 0x13, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x32, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x12, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x61, 0x13, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xd0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x61, 0x13, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xe0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x11, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x4f, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x1a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, + 0xb8, 0xff, 0xff, 0xff, 0xbf, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x24, 0x00, 0x07, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x21, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x69, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xec, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x71, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xee, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x19, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x71, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xf0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x16, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa1, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0xa1, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa2, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0xa1, 0xf6, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x73, 0x17, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x01, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x73, 0x17, 0x0b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0xa1, 0xf4, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x17, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x73, 0x17, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa1, 0xf2, 0xff, 0x00, 0x00, 0x00, 0x00, 0x73, 0x17, 0x06, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x73, 0x17, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x62, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0xfd, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x00, 0xfb, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x50, 0x4c, 0x00, + 0x9f, 0xeb, 0x01, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7c, 0x02, 0x00, 0x00, 0x7c, 0x02, 0x00, 0x00, 0xce, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x04, 0x20, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0e, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x04, 0x20, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x0f, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x12, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x04, + 0x18, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x14, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x0d, 0x02, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x0c, + 0x15, 0x00, 0x00, 0x00, 0xb2, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb7, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0e, 0x18, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x0a, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xc6, 0x0a, 0x00, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x69, 0x6e, 0x74, 0x00, 0x5f, 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, + 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5f, + 0x00, 0x74, 0x79, 0x70, 0x65, 0x00, 0x6d, 0x61, 0x78, 0x5f, 0x65, 0x6e, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x00, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x69, + 0x7a, 0x65, 0x00, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x00, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, + 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, 0x64, 0x70, 0x5f, + 0x6d, 0x64, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x65, 0x6e, 0x64, 0x00, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x00, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x69, + 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x72, 0x78, 0x5f, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x65, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x5f, 0x69, 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x00, 0x5f, 0x5f, 0x75, 0x33, 0x32, 0x00, 0x75, 0x6e, 0x73, 0x69, 0x67, + 0x6e, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x74, 0x00, 0x63, 0x74, 0x78, 0x00, + 0x78, 0x64, 0x70, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x5f, 0x64, 0x6e, 0x73, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x00, 0x78, 0x64, + 0x70, 0x00, 0x2f, 0x68, 0x6f, 0x6d, 0x65, 0x2f, 0x64, 0x73, 0x61, 0x6c, + 0x7a, 0x6d, 0x61, 0x6e, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x32, 0x2f, 0x73, + 0x72, 0x63, 0x2f, 0x6c, 0x69, 0x62, 0x6b, 0x6e, 0x6f, 0x74, 0x2f, 0x78, + 0x64, 0x70, 0x2f, 0x62, 0x70, 0x66, 0x2d, 0x6b, 0x65, 0x72, 0x6e, 0x65, + 0x6c, 0x2e, 0x63, 0x00, 0x69, 0x6e, 0x74, 0x20, 0x78, 0x64, 0x70, 0x5f, + 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x6e, 0x73, + 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x20, 0x78, 0x64, 0x70, 0x5f, 0x6d, 0x64, 0x20, 0x2a, 0x63, 0x74, 0x78, + 0x29, 0x00, 0x09, 0x5f, 0x5f, 0x75, 0x33, 0x32, 0x20, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x20, 0x3d, 0x20, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x72, 0x78, + 0x5f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x3b, 0x00, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x6b, 0x6e, + 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x6f, 0x70, 0x74, 0x73, 0x20, + 0x2a, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x70, 0x74, 0x72, 0x20, 0x3d, 0x20, + 0x62, 0x70, 0x66, 0x5f, 0x6d, 0x61, 0x70, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x28, 0x26, 0x6f, 0x70, 0x74, + 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x2c, 0x20, 0x26, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x29, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x21, 0x6f, 0x70, + 0x74, 0x73, 0x5f, 0x70, 0x74, 0x72, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x21, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, + 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, + 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x4f, 0x4e, + 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x29, + 0x62, 0x70, 0x66, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x61, 0x64, 0x6a, 0x75, + 0x73, 0x74, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x28, 0x63, 0x74, 0x78, 0x2c, + 0x20, 0x2d, 0x20, 0x28, 0x69, 0x6e, 0x74, 0x29, 0x73, 0x69, 0x7a, 0x65, + 0x6f, 0x66, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x6b, 0x6e, + 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x29, + 0x00, 0x09, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x64, 0x61, 0x74, 0x61, + 0x20, 0x3d, 0x20, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, + 0x6c, 0x6f, 0x6e, 0x67, 0x29, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, + 0x74, 0x61, 0x3b, 0x00, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, + 0x6b, 0x6e, 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x69, 0x6e, 0x66, + 0x6f, 0x20, 0x2a, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, 0x6c, 0x6f, 0x6e, 0x67, 0x29, + 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x2a, 0x29, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x6d, 0x65, 0x74, 0x61, + 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x2a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x3d, 0x20, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, 0x6c, 0x6f, 0x6e, + 0x67, 0x29, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x65, 0x6e, 0x64, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, + 0x72, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, + 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, 0x72, 0x29, 0x20, 0x3e, 0x20, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x69, 0x66, 0x20, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, 0x72, 0x2d, + 0x3e, 0x68, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x20, 0x3d, 0x3d, 0x20, + 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x68, + 0x74, 0x6f, 0x6e, 0x73, 0x28, 0x45, 0x54, 0x48, 0x5f, 0x50, 0x5f, 0x38, + 0x30, 0x32, 0x31, 0x51, 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, 0x73, 0x69, + 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x5f, 0x5f, 0x75, 0x31, 0x36, 0x29, 0x20, + 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x65, 0x74, 0x68, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x65, + 0x74, 0x61, 0x20, 0x3d, 0x3d, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x09, 0x5f, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x5f, 0x6d, + 0x65, 0x6d, 0x63, 0x70, 0x79, 0x28, 0x26, 0x65, 0x74, 0x68, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x5f, 0x5f, 0x75, 0x31, 0x36, + 0x29, 0x2c, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x65, 0x74, + 0x68, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x29, 0x29, 0x3b, 0x00, 0x09, 0x73, + 0x77, 0x69, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, + 0x28, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x69, 0x70, 0x34, + 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x69, + 0x70, 0x34, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, + 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, + 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x21, 0x3d, 0x20, 0x34, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, + 0x2d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3c, 0x20, 0x5f, 0x5f, 0x62, + 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x69, 0x70, 0x34, + 0x2d, 0x3e, 0x74, 0x6f, 0x74, 0x5f, 0x6c, 0x65, 0x6e, 0x29, 0x29, 0x20, + 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, 0x70, 0x34, 0x2d, + 0x3e, 0x66, 0x72, 0x61, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x20, 0x21, 0x3d, + 0x20, 0x30, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, 0x6c, 0x34, 0x5f, 0x68, + 0x64, 0x72, 0x20, 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x69, 0x68, 0x6c, 0x20, 0x2a, 0x20, 0x34, + 0x3b, 0x00, 0x09, 0x09, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x20, 0x3d, 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x3b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x69, 0x70, 0x36, 0x20, + 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x69, 0x70, + 0x36, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, + 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, + 0x70, 0x36, 0x2d, 0x3e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, + 0x21, 0x3d, 0x20, 0x36, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x2d, + 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3c, 0x20, 0x5f, 0x5f, 0x62, 0x70, + 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x69, 0x70, 0x36, 0x2d, + 0x3e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6c, 0x65, 0x6e, + 0x29, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, + 0x69, 0x70, 0x36, 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x70, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x36, + 0x2d, 0x3e, 0x6e, 0x65, 0x78, 0x74, 0x68, 0x64, 0x72, 0x3b, 0x00, 0x09, + 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x20, 0x3d, 0x3d, 0x20, 0x49, 0x50, 0x50, 0x52, 0x4f, 0x54, 0x4f, + 0x5f, 0x46, 0x52, 0x41, 0x47, 0x4d, 0x45, 0x4e, 0x54, 0x29, 0x20, 0x7b, + 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x2a, 0x29, 0x66, 0x72, 0x61, 0x67, 0x20, 0x2b, 0x20, 0x73, + 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x66, 0x72, 0x61, 0x67, 0x29, + 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, + 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x66, 0x72, 0x61, 0x67, 0x2d, 0x3e, 0x6e, + 0x65, 0x78, 0x74, 0x68, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x73, 0x77, 0x69, + 0x74, 0x63, 0x68, 0x20, 0x28, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6c, + 0x34, 0x5f, 0x68, 0x64, 0x72, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, + 0x6f, 0x66, 0x28, 0x2a, 0x74, 0x63, 0x70, 0x29, 0x20, 0x3e, 0x20, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, + 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, + 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x54, + 0x43, 0x50, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, 0x70, 0x6f, 0x72, + 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, 0x3d, 0x20, 0x5f, 0x5f, 0x62, + 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x74, 0x63, 0x70, + 0x2d, 0x3e, 0x64, 0x65, 0x73, 0x74, 0x29, 0x3b, 0x00, 0x09, 0x09, 0x20, + 0x20, 0x20, 0x20, 0x28, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x65, 0x73, + 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x75, 0x64, + 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x09, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x28, 0x4b, 0x4e, 0x4f, + 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, + 0x5f, 0x50, 0x41, 0x53, 0x53, 0x20, 0x7c, 0x20, 0x4b, 0x4e, 0x4f, 0x54, + 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, + 0x44, 0x52, 0x4f, 0x50, 0x29, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, + 0x69, 0x66, 0x20, 0x28, 0x6c, 0x34, 0x5f, 0x68, 0x64, 0x72, 0x20, 0x2b, + 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x75, 0x64, 0x70, + 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, + 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x2d, 0x20, 0x28, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x2a, 0x29, 0x75, 0x64, 0x70, 0x20, 0x3c, 0x20, 0x5f, + 0x5f, 0x62, 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x75, + 0x64, 0x70, 0x2d, 0x3e, 0x6c, 0x65, 0x6e, 0x29, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x09, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, + 0x3d, 0x20, 0x5f, 0x5f, 0x62, 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, + 0x73, 0x28, 0x75, 0x64, 0x70, 0x2d, 0x3e, 0x64, 0x65, 0x73, 0x74, 0x29, + 0x3b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, + 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, + 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, + 0x52, 0x5f, 0x55, 0x44, 0x50, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, + 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, + 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, + 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x51, 0x55, 0x49, 0x43, 0x29, 0x20, 0x26, + 0x26, 0x00, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, 0x28, 0x70, 0x6f, 0x72, + 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x6f, 0x70, + 0x74, 0x73, 0x2e, 0x71, 0x75, 0x69, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, + 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, + 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x44, 0x52, 0x4f, + 0x50, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6f, 0x70, + 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, + 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, + 0x45, 0x52, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x62, 0x70, 0x66, + 0x5f, 0x66, 0x69, 0x62, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x20, + 0x66, 0x69, 0x62, 0x20, 0x3d, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x69, 0x70, 0x76, 0x34, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, + 0x09, 0x66, 0x69, 0x62, 0x2e, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x20, + 0x20, 0x20, 0x3d, 0x20, 0x41, 0x46, 0x5f, 0x49, 0x4e, 0x45, 0x54, 0x3b, + 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, 0x62, 0x2e, 0x69, 0x70, 0x76, 0x34, + 0x5f, 0x73, 0x72, 0x63, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, + 0x64, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, + 0x62, 0x2e, 0x69, 0x70, 0x76, 0x34, 0x5f, 0x64, 0x73, 0x74, 0x20, 0x3d, + 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x73, 0x61, 0x64, 0x64, 0x72, 0x3b, + 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, 0x62, 0x2e, 0x66, 0x61, 0x6d, 0x69, + 0x6c, 0x79, 0x20, 0x3d, 0x20, 0x41, 0x46, 0x5f, 0x49, 0x4e, 0x45, 0x54, + 0x36, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x2a, 0x69, 0x70, 0x76, 0x36, 0x5f, + 0x73, 0x72, 0x63, 0x20, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x36, 0x2d, 0x3e, + 0x64, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x2a, 0x69, + 0x70, 0x76, 0x36, 0x5f, 0x64, 0x73, 0x74, 0x20, 0x20, 0x3d, 0x20, 0x69, + 0x70, 0x36, 0x2d, 0x3e, 0x73, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, + 0x09, 0x69, 0x6e, 0x74, 0x20, 0x72, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x62, + 0x70, 0x66, 0x5f, 0x66, 0x69, 0x62, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, + 0x70, 0x28, 0x63, 0x74, 0x78, 0x2c, 0x20, 0x26, 0x66, 0x69, 0x62, 0x2c, + 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x66, 0x69, 0x62, 0x29, + 0x2c, 0x20, 0x42, 0x50, 0x46, 0x5f, 0x46, 0x49, 0x42, 0x5f, 0x4c, 0x4f, + 0x4f, 0x4b, 0x55, 0x50, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x29, + 0x3b, 0x00, 0x09, 0x09, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68, 0x20, 0x28, + 0x72, 0x65, 0x74, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x6d, 0x61, 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x30, 0x5d, 0x20, + 0x21, 0x3d, 0x20, 0x6d, 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x30, + 0x5d, 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, + 0x6d, 0x61, 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x31, 0x5d, 0x20, 0x21, 0x3d, + 0x20, 0x6d, 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x31, 0x5d, 0x20, + 0x7c, 0x7c, 0x00, 0x09, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, + 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x32, 0x5d, 0x20, 0x21, 0x3d, 0x20, 0x6d, + 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x32, 0x5d, 0x29, 0x20, 0x7b, + 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x65, 0x74, 0x61, + 0x20, 0x21, 0x3d, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, + 0x09, 0x6d, 0x65, 0x74, 0x61, 0x2d, 0x3e, 0x6f, 0x75, 0x74, 0x5f, 0x69, + 0x66, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x20, 0x3d, 0x20, 0x66, 0x69, + 0x62, 0x2e, 0x69, 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x3b, 0x00, 0x09, + 0x09, 0x09, 0x5f, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x5f, + 0x6d, 0x65, 0x6d, 0x63, 0x70, 0x79, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x68, + 0x64, 0x72, 0x2d, 0x3e, 0x68, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2c, 0x20, 0x66, 0x69, 0x62, 0x2e, 0x64, 0x6d, 0x61, 0x63, 0x2c, 0x20, + 0x45, 0x54, 0x48, 0x5f, 0x41, 0x4c, 0x45, 0x4e, 0x29, 0x3b, 0x00, 0x09, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x62, 0x70, 0x66, 0x5f, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x6d, 0x61, 0x70, 0x28, + 0x26, 0x78, 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x2c, 0x20, 0x63, + 0x74, 0x78, 0x2d, 0x3e, 0x72, 0x78, 0x5f, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2c, 0x20, 0x30, 0x29, 0x3b, 0x00, + 0x7d, 0x00, 0x63, 0x68, 0x61, 0x72, 0x00, 0x5f, 0x6c, 0x69, 0x63, 0x65, + 0x6e, 0x73, 0x65, 0x00, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x00, 0x6c, 0x69, + 0x63, 0x65, 0x6e, 0x73, 0x65, 0x00, 0x00, 0x00, 0x9f, 0xeb, 0x01, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x2c, 0x07, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0xf0, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x2e, 0x01, 0x00, 0x00, 0x15, 0xfc, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, 0x08, 0xfc, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x52, 0x01, 0x00, 0x00, 0x23, 0x00, 0x01, 0x00, 0x50, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, 0x06, 0x04, 0x01, 0x00, + 0x58, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, + 0x08, 0x1c, 0x01, 0x00, 0x78, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xae, 0x01, 0x00, 0x00, 0x13, 0x1c, 0x01, 0x00, 0x90, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0x06, 0x1c, 0x01, 0x00, + 0x98, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xd9, 0x01, 0x00, 0x00, 0x08, 0x30, 0x01, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, 0x22, 0x3c, 0x01, 0x00, + 0xf8, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x44, 0x02, 0x00, 0x00, + 0x32, 0x44, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x80, 0x02, 0x00, 0x00, 0x13, 0x50, 0x01, 0x00, 0x18, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x06, 0x50, 0x01, 0x00, + 0x30, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xac, 0x02, 0x00, 0x00, + 0x2c, 0x40, 0x01, 0x00, 0x38, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xe1, 0x02, 0x00, 0x00, 0x16, 0x8c, 0x01, 0x00, 0x50, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xe1, 0x02, 0x00, 0x00, 0x06, 0x8c, 0x01, 0x00, + 0x58, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, + 0x0f, 0xa4, 0x01, 0x00, 0x78, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x17, 0x03, 0x00, 0x00, 0x06, 0xa4, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, 0x1c, 0xa8, 0x01, 0x00, + 0x98, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, + 0x07, 0xa8, 0x01, 0x00, 0xa8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x8d, 0x03, 0x00, 0x00, 0x0e, 0xb0, 0x01, 0x00, 0xb8, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xa7, 0x03, 0x00, 0x00, 0x03, 0xbc, 0x01, 0x00, + 0xd8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, + 0x02, 0xe0, 0x01, 0x00, 0xf8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x03, 0x04, 0x00, 0x00, 0x13, 0xec, 0x01, 0x00, 0x10, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x07, 0xec, 0x01, 0x00, + 0x18, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x32, 0x04, 0x00, 0x00, + 0x0c, 0xf8, 0x01, 0x00, 0x20, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x32, 0x04, 0x00, 0x00, 0x14, 0xf8, 0x01, 0x00, 0x38, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x32, 0x04, 0x00, 0x00, 0x07, 0xf8, 0x01, 0x00, + 0x40, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x4d, 0x04, 0x00, 0x00, + 0x10, 0x10, 0x02, 0x00, 0x50, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x4d, 0x04, 0x00, 0x00, 0x19, 0x10, 0x02, 0x00, 0x68, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x4d, 0x04, 0x00, 0x00, 0x07, 0x10, 0x02, 0x00, + 0x70, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x82, 0x04, 0x00, 0x00, + 0x0c, 0x20, 0x02, 0x00, 0x78, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x82, 0x04, 0x00, 0x00, 0x1a, 0x20, 0x02, 0x00, 0xa8, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, 0x1c, 0x34, 0x02, 0x00, + 0xb8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, + 0x11, 0x34, 0x02, 0x00, 0xc8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xbe, 0x04, 0x00, 0x00, 0x13, 0x30, 0x02, 0x00, 0xd8, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00, 0x13, 0x48, 0x02, 0x00, + 0xf0, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00, + 0x07, 0x48, 0x02, 0x00, 0xf8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x09, 0x05, 0x00, 0x00, 0x0c, 0x54, 0x02, 0x00, 0x00, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, 0x14, 0x54, 0x02, 0x00, + 0x10, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, + 0x07, 0x54, 0x02, 0x00, 0x18, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0x10, 0x6c, 0x02, 0x00, 0x28, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, 0x19, 0x6c, 0x02, 0x00, + 0x38, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, + 0x37, 0x6c, 0x02, 0x00, 0x48, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0x07, 0x6c, 0x02, 0x00, 0x68, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x6c, 0x05, 0x00, 0x00, 0x13, 0x7c, 0x02, 0x00, + 0x70, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x87, 0x05, 0x00, 0x00, + 0x07, 0x84, 0x02, 0x00, 0x80, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xad, 0x05, 0x00, 0x00, 0x15, 0x90, 0x02, 0x00, 0x90, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xad, 0x05, 0x00, 0x00, 0x08, 0x90, 0x02, 0x00, + 0xa0, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xdf, 0x05, 0x00, 0x00, + 0x15, 0x9c, 0x02, 0x00, 0xb8, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xfc, 0x05, 0x00, 0x00, 0x02, 0xe0, 0x02, 0x00, + 0xf0, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x11, 0x06, 0x00, 0x00, + 0x0e, 0xf0, 0x02, 0x00, 0x08, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x11, 0x06, 0x00, 0x00, 0x07, 0xf0, 0x02, 0x00, 0x10, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, 0x13, 0x08, 0x03, 0x00, + 0x28, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, + 0x2a, 0x08, 0x03, 0x00, 0x30, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x67, 0x06, 0x00, 0x00, 0x0f, 0x00, 0x03, 0x00, 0x40, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, 0x12, 0x0c, 0x03, 0x00, + 0x50, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, + 0x23, 0x0c, 0x03, 0x00, 0x58, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb2, 0x06, 0x00, 0x00, 0x15, 0x10, 0x03, 0x00, 0x70, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x46, 0x10, 0x03, 0x00, + 0x98, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xfa, 0x06, 0x00, 0x00, + 0x0e, 0x30, 0x03, 0x00, 0xb0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xfa, 0x06, 0x00, 0x00, 0x07, 0x30, 0x03, 0x00, 0xb8, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, 0x10, 0x44, 0x03, 0x00, + 0xc0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, + 0x20, 0x44, 0x03, 0x00, 0xd8, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x07, 0x00, 0x00, 0x07, 0x44, 0x03, 0x00, 0xe0, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x5c, 0x07, 0x00, 0x00, 0x0f, 0x54, 0x03, 0x00, + 0xf0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x82, 0x07, 0x00, 0x00, + 0x13, 0x5c, 0x03, 0x00, 0x00, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x82, 0x07, 0x00, 0x00, 0x2a, 0x5c, 0x03, 0x00, 0x08, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, 0x12, 0x60, 0x03, 0x00, + 0x18, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, + 0x23, 0x60, 0x03, 0x00, 0x20, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb2, 0x06, 0x00, 0x00, 0x15, 0x64, 0x03, 0x00, 0x30, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x46, 0x64, 0x03, 0x00, + 0x48, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xae, 0x07, 0x00, 0x00, + 0x1a, 0x70, 0x03, 0x00, 0x60, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xae, 0x07, 0x00, 0x00, 0x32, 0x70, 0x03, 0x00, 0x70, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xe2, 0x07, 0x00, 0x00, + 0x12, 0x74, 0x03, 0x00, 0x98, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xe2, 0x07, 0x00, 0x00, 0x24, 0x74, 0x03, 0x00, 0xa0, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x15, 0x78, 0x03, 0x00, + 0xb8, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, + 0x46, 0x78, 0x03, 0x00, 0xd8, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x00, 0x00, 0x18, 0xac, 0x03, 0x00, 0xf0, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x08, 0x08, 0x00, 0x00, 0x0d, 0xac, 0x03, 0x00, + 0x10, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x39, 0x08, 0x00, 0x00, + 0x11, 0xd0, 0x03, 0x00, 0x18, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x39, 0x08, 0x00, 0x00, 0x06, 0xd0, 0x03, 0x00, 0x28, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x64, 0x08, 0x00, 0x00, 0x19, 0xd4, 0x03, 0x00, + 0x78, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x84, 0x08, 0x00, 0x00, + 0x07, 0xe0, 0x03, 0x00, 0x90, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x92, 0x08, 0x00, 0x00, 0x11, 0xe4, 0x03, 0x00, 0x98, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xad, 0x08, 0x00, 0x00, 0x18, 0xe8, 0x03, 0x00, + 0xa0, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xad, 0x08, 0x00, 0x00, + 0x11, 0xe8, 0x03, 0x00, 0xa8, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xcb, 0x08, 0x00, 0x00, 0x18, 0xec, 0x03, 0x00, 0xb0, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xcb, 0x08, 0x00, 0x00, 0x11, 0xec, 0x03, 0x00, + 0xc8, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xe9, 0x08, 0x00, 0x00, + 0x0f, 0xfc, 0x03, 0x00, 0xd0, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x03, 0x09, 0x00, 0x00, 0x16, 0x00, 0x04, 0x00, 0x20, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x1f, 0x09, 0x00, 0x00, 0x16, 0x04, 0x04, 0x00, + 0x78, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x3b, 0x09, 0x00, 0x00, 0x0d, 0x18, 0x04, 0x00, 0xb8, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x86, 0x09, 0x00, 0x00, 0x03, 0x1c, 0x04, 0x00, + 0xe0, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, + 0x08, 0x28, 0x04, 0x00, 0xe8, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x97, 0x09, 0x00, 0x00, 0x15, 0x28, 0x04, 0x00, 0xf0, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, 0x20, 0x28, 0x04, 0x00, + 0xf8, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xb9, 0x09, 0x00, 0x00, + 0x08, 0x2c, 0x04, 0x00, 0x00, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb9, 0x09, 0x00, 0x00, 0x15, 0x2c, 0x04, 0x00, 0x08, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb9, 0x09, 0x00, 0x00, 0x20, 0x2c, 0x04, 0x00, + 0x10, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xdb, 0x09, 0x00, 0x00, + 0x08, 0x30, 0x04, 0x00, 0x18, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xdb, 0x09, 0x00, 0x00, 0x15, 0x30, 0x04, 0x00, 0x20, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, 0x08, 0x28, 0x04, 0x00, + 0x28, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xfd, 0x09, 0x00, 0x00, + 0x08, 0x44, 0x04, 0x00, 0x38, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x11, 0x0a, 0x00, 0x00, 0x1e, 0x48, 0x04, 0x00, 0x40, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x11, 0x0a, 0x00, 0x00, 0x18, 0x48, 0x04, 0x00, + 0x50, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x37, 0x0a, 0x00, 0x00, + 0x04, 0x58, 0x04, 0x00, 0xb0, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x73, 0x0a, 0x00, 0x00, 0x09, 0x88, 0x04, 0x00, 0xd8, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb0, 0x0a, 0x00, 0x00, 0x01, 0x8c, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x28, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xa8, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xb8, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x98, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x48, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xc0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x70, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xe0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x50, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4b, 0x00, 0x00, 0x00, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x11, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2b, 0x00, 0x00, 0x00, 0x11, 0x00, 0x05, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x00, 0x00, 0x00, 0x11, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0xb8, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x68, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, + 0x74, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x8c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x60, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x60, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa0, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xb0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd0, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xe0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x30, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x60, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x70, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x90, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xa0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb0, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xd0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe0, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x30, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x50, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x60, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x90, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa0, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xb0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd0, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xe0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xf0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x30, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x1a, 0x1b, 0x1c, 0x1d, 0x00, 0x2e, 0x74, 0x65, 0x78, 0x74, 0x00, 0x2e, + 0x72, 0x65, 0x6c, 0x2e, 0x42, 0x54, 0x46, 0x2e, 0x65, 0x78, 0x74, 0x00, + 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x00, 0x2e, 0x72, 0x65, 0x6c, 0x78, 0x64, + 0x70, 0x00, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, + 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x2e, 0x6c, 0x6c, 0x76, + 0x6d, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x69, 0x67, 0x00, 0x5f, 0x6c, + 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x00, 0x78, 0x64, 0x70, 0x5f, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x6e, 0x73, 0x5f, + 0x66, 0x75, 0x6e, 0x63, 0x00, 0x2e, 0x73, 0x74, 0x72, 0x74, 0x61, 0x62, + 0x00, 0x2e, 0x73, 0x79, 0x6d, 0x74, 0x61, 0x62, 0x00, 0x2e, 0x72, 0x65, + 0x6c, 0x2e, 0x42, 0x54, 0x46, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x39, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x35, 0x38, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x35, 0x37, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x31, 0x37, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x35, 0x36, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x34, 0x36, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x33, 0x36, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x31, 0x36, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x35, 0x35, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x35, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x00, 0x4c, 0x42, 0x42, 0x30, + 0x5f, 0x35, 0x34, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x32, 0x33, 0x00, + 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x30, 0x00, 0x4c, 0x42, 0x42, 0x30, + 0x5f, 0x33, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc4, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1a, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x09, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x0d, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x16, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x03, 0x4c, 0xff, 0x6f, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +unsigned int bpf_kernel_o_len = 11256; diff --git a/src/libknot/xdp/bpf-kernel-obj.h b/src/libknot/xdp/bpf-kernel-obj.h new file mode 100644 index 0000000..7c3d759 --- /dev/null +++ b/src/libknot/xdp/bpf-kernel-obj.h @@ -0,0 +1,2 @@ +extern unsigned char bpf_kernel_o[]; +extern unsigned int bpf_kernel_o_len; diff --git a/src/libknot/xdp/bpf-kernel.c b/src/libknot/xdp/bpf-kernel.c new file mode 100644 index 0000000..8bffd7f --- /dev/null +++ b/src/libknot/xdp/bpf-kernel.c @@ -0,0 +1,293 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/udp.h> + +#include "bpf-consts.h" +#include "../../contrib/libbpf/include/uapi/linux/bpf.h" +#include "../../contrib/libbpf/bpf/bpf_endian.h" +#include "../../contrib/libbpf/bpf/bpf_helpers.h" + +/* Don't fragment flag. */ +#define IP_DF 0x4000 + +#define AF_INET 2 +#define AF_INET6 10 + +/* Define maximum reasonable number of NIC queues supported. */ +#define QUEUE_MAX 256 + +/* A map of configuration options. */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, QUEUE_MAX); + __uint(key_size, sizeof(__u32)); /* Must be 4 bytes. */ + __uint(value_size, sizeof(knot_xdp_opts_t)); +} opts_map SEC(".maps"); + +/* A map of AF_XDP sockets. */ +struct { + __uint(type, BPF_MAP_TYPE_XSKMAP); + __uint(max_entries, QUEUE_MAX); + __uint(key_size, sizeof(__u32)); /* Must be 4 bytes. */ + __uint(value_size, sizeof(int)); +} xsks_map SEC(".maps"); + +struct ipv6_frag_hdr { + unsigned char nexthdr; + unsigned char whatever[7]; +} __attribute__((packed)); + +SEC("xdp") +int xdp_redirect_dns_func(struct xdp_md *ctx) +{ + /* Get the queue options. */ + __u32 index = ctx->rx_queue_index; + struct knot_xdp_opts *opts_ptr = bpf_map_lookup_elem(&opts_map, &index); + if (!opts_ptr) { + return XDP_ABORTED; + } + knot_xdp_opts_t opts = *opts_ptr; + + /* Check if the filter is disabled. */ + if (!(opts.flags & KNOT_XDP_FILTER_ON)) { + return XDP_PASS; + } + + /* Try to reserve space in front of the packet for additional (VLAN) data. */ + (void)bpf_xdp_adjust_meta(ctx, - (int)sizeof(struct knot_xdp_info) + - KNOT_XDP_PKT_ALIGNMENT); + + void *data = (void *)(long)ctx->data; + const void *data_end = (void *)(long)ctx->data_end; + struct knot_xdp_info *meta = (void *)(long)ctx->data_meta; + + /* Check if the meta data pointer is usable (e.g. not `tap` interface). */ + if ((void *)meta + sizeof(*meta) > data) { + meta = 0; + } + + struct ethhdr *eth_hdr = data; + const void *ip_hdr; + const struct iphdr *ip4; + const struct ipv6hdr *ip6; + const void *l4_hdr; + __u8 ipv4; + __u8 ip_proto; + __u8 fragmented = 0; + __u16 eth_type; /* In big endian. */ + + /* Parse Ethernet header. */ + if ((void *)eth_hdr + sizeof(*eth_hdr) > data_end) { + return XDP_DROP; + } + data += sizeof(*eth_hdr); + + /* Parse possible VLAN (802.1Q) header. */ + if (eth_hdr->h_proto == __constant_htons(ETH_P_8021Q)) { + if (data + sizeof(__u16) + sizeof(eth_type) > data_end) { + return XDP_DROP; + } else if (meta == 0) { /* VLAN not supported. */ + return XDP_PASS; + } + __builtin_memcpy(ð_type, data + sizeof(__u16), sizeof(eth_type)); + data += sizeof(__u16) + sizeof(eth_type); + } else { + eth_type = eth_hdr->h_proto; + } + + ip_hdr = data; + + /* Parse IPv4 or IPv6 header. */ + switch (eth_type) { + case __constant_htons(ETH_P_IP): + ip4 = ip_hdr; + if ((void *)ip4 + sizeof(*ip4) > data_end) { + return XDP_DROP; + } + if (ip4->version != 4) { + return XDP_DROP; + } + + /* Check the IP length. Cannot use strict equality due to + * Ethernet padding applied to frames shorter than 64 octects. */ + if (data_end - data < __bpf_ntohs(ip4->tot_len)) { + return XDP_DROP; + } + + if (ip4->frag_off != 0 && + ip4->frag_off != __constant_htons(IP_DF)) { + fragmented = 1; + } + ip_proto = ip4->protocol; + l4_hdr = data + ip4->ihl * 4; + ipv4 = 1; + break; + case __constant_htons(ETH_P_IPV6): + ip6 = ip_hdr; + if ((void *)ip6 + sizeof(*ip6) > data_end) { + return XDP_DROP; + } + if (ip6->version != 6) { + return XDP_DROP; + } + + /* Check the IP length. Cannot use strict equality due to + * Ethernet padding applied to frames shorter than 64 octects. */ + if (data_end - data < __bpf_ntohs(ip6->payload_len) + sizeof(*ip6)) { + return XDP_DROP; + } + + ip_proto = ip6->nexthdr; + data += sizeof(*ip6); + if (ip_proto == IPPROTO_FRAGMENT) { + fragmented = 1; + const struct ipv6_frag_hdr *frag = data; + if ((void *)frag + sizeof(*frag) > data_end) { + return XDP_DROP; + } + ip_proto = frag->nexthdr; + data += sizeof(*frag); + } + l4_hdr = data; + ipv4 = 0; + break; + default: + /* Pass packets of possible other protocols. */ + return XDP_PASS; + } + + const struct tcphdr *tcp; + const struct udphdr *udp; + __u16 port_dest; + __u8 match = 0; + + /* Check the transport protocol. */ + switch (ip_proto) { + case IPPROTO_TCP: + /* Parse TCP header. */ + tcp = l4_hdr; + if (l4_hdr + sizeof(*tcp) > data_end) { + return XDP_DROP; + } + + port_dest = __bpf_ntohs(tcp->dest); + + if ((opts.flags & KNOT_XDP_FILTER_TCP) && + (port_dest == opts.udp_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.udp_port))) { + match = 1; + } + break; + case IPPROTO_UDP: + /* Parse UDP header. */ + udp = l4_hdr; + if (l4_hdr + sizeof(*udp) > data_end) { + return XDP_DROP; + } + + /* Check the UDP length. */ + if (data_end - (void *)udp < __bpf_ntohs(udp->len)) { + return XDP_DROP; + } + + port_dest = __bpf_ntohs(udp->dest); + + if ((opts.flags & KNOT_XDP_FILTER_UDP) && + (port_dest == opts.udp_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.udp_port))) { + match = 1; + } else if ((opts.flags & KNOT_XDP_FILTER_QUIC) && + (port_dest == opts.quic_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.quic_port))) { + match = 1; + } + break; + default: + /* Pass packets of possible other protocols. */ + return XDP_PASS; + } + + if (!match) { + /* Pass non-matching packet. */ + return XDP_PASS; + } else if (opts.flags & KNOT_XDP_FILTER_DROP) { + /* Drop matching packet if requested. */ + return XDP_DROP; + } else if (fragmented) { + /* Drop fragmented packet. */ + return XDP_DROP; + } + + /* Take into account routing information. */ + if (opts.flags & KNOT_XDP_FILTER_ROUTE) { + struct bpf_fib_lookup fib = { + .ifindex = 1 /* Loopback. */ + }; + if (ipv4) { + fib.family = AF_INET; + fib.ipv4_src = ip4->daddr; + fib.ipv4_dst = ip4->saddr; + } else { + struct in6_addr *ipv6_src = (struct in6_addr *)fib.ipv6_src; + struct in6_addr *ipv6_dst = (struct in6_addr *)fib.ipv6_dst; + fib.family = AF_INET6; + *ipv6_src = ip6->daddr; + *ipv6_dst = ip6->saddr; + } + + const __u16 *mac_in = (const __u16 *)eth_hdr->h_dest; + const __u16 *mac_out = (const __u16 *)fib.smac; + int ret = bpf_fib_lookup(ctx, &fib, sizeof(fib), BPF_FIB_LOOKUP_DIRECT); + switch (ret) { + case BPF_FIB_LKUP_RET_SUCCESS: + /* Cross-interface answers are handled through normal stack. */ + if (mac_in[0] != mac_out[0] || + mac_in[1] != mac_out[1] || + mac_in[2] != mac_out[2]) { + return XDP_PASS; + } + + /* Store output interface index for later use with VLAN in user space. */ + if (meta != 0) { + meta->out_if_index = fib.ifindex; + } + + /* Update destination MAC for responding. */ + __builtin_memcpy(eth_hdr->h_source, fib.dmac, ETH_ALEN); + break; + case BPF_FIB_LKUP_RET_FWD_DISABLED: /* Disabled forwarding on loopback. */ + return XDP_ABORTED; + case BPF_FIB_LKUP_RET_NO_NEIGH: /* Use normal stack to obtain MAC. */ + return XDP_PASS; + default: + return XDP_DROP; + } + } + + /* Forward the packet to user space. */ + return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, 0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/src/libknot/xdp/bpf-user.c b/src/libknot/xdp/bpf-user.c new file mode 100644 index 0000000..a0c8bae --- /dev/null +++ b/src/libknot/xdp/bpf-user.c @@ -0,0 +1,313 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <bpf/bpf.h> +#include <linux/if_link.h> +#include <net/if.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "libknot/endian.h" +#include "libknot/error.h" +#include "libknot/xdp/bpf-kernel-obj.h" +#include "libknot/xdp/bpf-user.h" +#include "libknot/xdp/eth.h" +#include "contrib/openbsd/strlcpy.h" + +#define NO_BPF_MAPS 2 + +static inline bool IS_ERR_OR_NULL(const void *ptr) +{ + return (ptr == NULL) || (unsigned long)ptr >= (unsigned long)-4095; +} + +static int prog_load(struct bpf_object **pobj, int *prog_fd) +{ + struct bpf_program *prog, *first_prog = NULL; + struct bpf_object *obj; + + obj = bpf_object__open_mem(bpf_kernel_o, bpf_kernel_o_len, NULL); + if (IS_ERR_OR_NULL(obj)) { + return KNOT_ENOENT; + } + + bpf_object__for_each_program(prog, obj) { + bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); + if (first_prog == NULL) { + first_prog = prog; + } + } + + if (first_prog == NULL) { + bpf_object__close(obj); + return KNOT_ENOENT; + } + + int ret = bpf_object__load(obj); + if (ret != 0) { + bpf_object__close(obj); + return KNOT_EINVAL; + } + + *pobj = obj; + *prog_fd = bpf_program__fd(first_prog); + + return KNOT_EOK; +} + +static int ensure_prog(struct kxsk_iface *iface, bool overwrite) +{ + if (bpf_kernel_o_len < 2) { + return KNOT_ENOTSUP; + } + + /* Use libbpf for extracting BPF byte-code from BPF-ELF object, and + * loading this into the kernel via bpf-syscall. */ + int prog_fd; + int ret = prog_load(&iface->prog_obj, &prog_fd); + if (ret != KNOT_EOK) { + return KNOT_EPROGRAM; + } + +#if USE_LIBXDP + ret = bpf_xdp_attach(iface->if_index, prog_fd, + overwrite ? 0 : XDP_FLAGS_UPDATE_IF_NOEXIST, NULL); +#else + ret = bpf_set_link_xdp_fd(iface->if_index, prog_fd, + overwrite ? 0 : XDP_FLAGS_UPDATE_IF_NOEXIST); +#endif + if (ret != 0) { + close(prog_fd); + } + if (ret == -EBUSY && !overwrite) { // We try accepting the present program. + uint32_t prog_id = 0; +#if USE_LIBXDP + ret = bpf_xdp_query_id(iface->if_index, 0, &prog_id); +#else + ret = bpf_get_link_xdp_id(iface->if_index, &prog_id, 0); +#endif + if (ret == 0 && prog_id != 0) { + ret = prog_fd = bpf_prog_get_fd_by_id(prog_id); + } + } + if (ret < 0) { + return KNOT_EFD; + } else { + return prog_fd; + } +} + +static void unget_bpf_maps(struct kxsk_iface *iface) +{ + if (iface->opts_map_fd >= 0) { + close(iface->opts_map_fd); + } + if (iface->xsks_map_fd >= 0) { + close(iface->xsks_map_fd); + } + iface->opts_map_fd = iface->xsks_map_fd = -1; +} + +/*! + * /brief Get FDs for the two maps and assign them into xsk_info-> fields. + * + * Inspired by xsk_lookup_bpf_maps() from libbpf before qidconf_map elimination. + */ +static int get_bpf_maps(int prog_fd, struct kxsk_iface *iface) +{ + uint32_t *map_ids = calloc(NO_BPF_MAPS, sizeof(*map_ids)); + if (map_ids == NULL) { + return KNOT_ENOMEM; + } + + struct bpf_prog_info prog_info = { + .nr_map_ids = NO_BPF_MAPS, + .map_ids = (__u64)(unsigned long)map_ids, + }; + + uint32_t prog_len = sizeof(struct bpf_prog_info); + int ret = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &prog_len); + if (ret != 0) { + free(map_ids); + return ret; + } + + for (int i = 0; i < NO_BPF_MAPS; ++i) { + int fd = bpf_map_get_fd_by_id(map_ids[i]); + if (fd < 0) { + continue; + } + + struct bpf_map_info map_info = { 0 }; + uint32_t map_len = sizeof(struct bpf_map_info); + ret = bpf_obj_get_info_by_fd(fd, &map_info, &map_len); + if (ret != 0) { + close(fd); + continue; + } + + if (strcmp(map_info.name, "opts_map") == 0) { + iface->opts_map_fd = fd; + continue; + } + + if (strcmp(map_info.name, "xsks_map") == 0) { + iface->xsks_map_fd = fd; + continue; + } + + close(fd); + } + + if (iface->opts_map_fd < 0 || iface->xsks_map_fd < 0) { + unget_bpf_maps(iface); + free(map_ids); + return KNOT_ENOENT; + } + + free(map_ids); + return KNOT_EOK; +} + +int kxsk_socket_start(const struct kxsk_iface *iface, knot_xdp_filter_flag_t flags, + uint16_t udp_port, uint16_t quic_port, struct xsk_socket *xsk) +{ + if (iface == NULL || xsk == NULL) { + return KNOT_EINVAL; + } + + int fd = xsk_socket__fd(xsk); + int ret = bpf_map_update_elem(iface->xsks_map_fd, &iface->if_queue, &fd, 0); + if (ret != 0) { + return ret; + } + + knot_xdp_opts_t opts = { + .flags = flags | KNOT_XDP_FILTER_ON, + .udp_port = udp_port, + .quic_port = quic_port, + }; + + ret = bpf_map_update_elem(iface->opts_map_fd, &iface->if_queue, &opts, 0); + if (ret != 0) { + (void)bpf_map_delete_elem(iface->xsks_map_fd, &iface->if_queue); + } + + return ret; +} + +void kxsk_socket_stop(const struct kxsk_iface *iface) +{ + if (iface == NULL) { + return; + } + + knot_xdp_opts_t opts = { 0 }; + + (void)bpf_map_update_elem(iface->opts_map_fd, &iface->if_queue, &opts, 0); + (void)bpf_map_delete_elem(iface->xsks_map_fd, &iface->if_queue); +} + +int kxsk_iface_new(const char *if_name, unsigned if_queue, knot_xdp_load_bpf_t load_bpf, + struct kxsk_iface **out_iface) +{ + if (if_name == NULL || out_iface == NULL) { + return KNOT_EINVAL; + } + + struct kxsk_iface *iface = calloc(1, sizeof(*iface) + IFNAMSIZ); + if (iface == NULL) { + return KNOT_ENOMEM; + } + iface->if_name = (char *)(iface + 1); + strlcpy((char *)iface->if_name, if_name, IFNAMSIZ); + iface->if_index = if_nametoindex(if_name); + if (iface->if_index == 0) { + free(iface); + return KNOT_EINVAL; + } + iface->if_queue = if_queue; + iface->opts_map_fd = iface->xsks_map_fd = -1; + + int ret; + switch (load_bpf) { + case KNOT_XDP_LOAD_BPF_NEVER: + (void)0; + uint32_t prog_id = 0; +#if USE_LIBXDP + ret = bpf_xdp_query_id(iface->if_index, 0, &prog_id); +#else + ret = bpf_get_link_xdp_id(iface->if_index, &prog_id, 0); +#endif + if (ret == 0) { + if (prog_id == 0) { + ret = KNOT_EPROGRAM; + } else { + ret = bpf_prog_get_fd_by_id(prog_id); + } + } + break; + case KNOT_XDP_LOAD_BPF_ALWAYS_UNLOAD: +#if USE_LIBXDP + (void)bpf_xdp_detach(iface->if_index, 0, NULL); +#else + (void)bpf_set_link_xdp_fd(iface->if_index, -1, 0); +#endif + sleep(1); + // FALLTHROUGH + case KNOT_XDP_LOAD_BPF_ALWAYS: + ret = ensure_prog(iface, true); + break; + case KNOT_XDP_LOAD_BPF_MAYBE: + ret = ensure_prog(iface, false); + break; + default: + return KNOT_EINVAL; + } + + if (ret >= 0) { + ret = get_bpf_maps(ret, iface); + } + if (ret < 0) { + free(iface); + return ret; + } + + knot_xdp_mode_t mode = knot_eth_xdp_mode(iface->if_index); + if (mode == KNOT_XDP_MODE_NONE) { + free(iface); + return KNOT_ENOTSUP; + } + + *out_iface = iface; + return KNOT_EOK; +} + +void kxsk_iface_free(struct kxsk_iface *iface) +{ + if (iface == NULL) { + return; + } + + unget_bpf_maps(iface); + + if (iface->prog_obj != NULL) { + bpf_object__close(iface->prog_obj); + } + + free(iface); +} diff --git a/src/libknot/xdp/bpf-user.h b/src/libknot/xdp/bpf-user.h new file mode 100644 index 0000000..14c54c5 --- /dev/null +++ b/src/libknot/xdp/bpf-user.h @@ -0,0 +1,139 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP socket interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#if USE_LIBXDP + #include <xdp/xsk.h> +#else + #include <bpf/xsk.h> +#endif + +#include "libknot/xdp/xdp.h" + +struct kxsk_iface { + /*! Interface name. */ + const char *if_name; + /*! Interface name index (derived from ifname). */ + int if_index; + /*! Network card queue id. */ + unsigned if_queue; + + /*! Configuration BPF map file descriptor. */ + int opts_map_fd; + /*! XSK BPF map file descriptor. */ + int xsks_map_fd; + + /*! BPF program object. */ + struct bpf_object *prog_obj; +}; + +struct kxsk_umem { + /*! Fill queue: passing memory frames to kernel - ready to receive. */ + struct xsk_ring_prod fq; + /*! Completion queue: passing memory frames from kernel - after send finishes. */ + struct xsk_ring_cons cq; + /*! Handle internal to libbpf. */ + struct xsk_umem *umem; + + /*! The memory frames. */ + struct umem_frame *frames; + /*! The number of free frames (for TX). */ + uint32_t tx_free_count; + /*! Stack of indices of the free frames (for TX). */ + uint16_t tx_free_indices[]; +}; + +struct knot_xdp_socket { + /*! Receive queue: passing arrived packets from kernel. */ + struct xsk_ring_cons rx; + /*! Transmit queue: passing packets to kernel for sending. */ + struct xsk_ring_prod tx; + /*! Information about memory frames for all the passed packets. */ + struct kxsk_umem *umem; + /*! Handle internal to libbpf. */ + struct xsk_socket *xsk; + + /*! Interface context. */ + const struct kxsk_iface *iface; + + /*! If non-NULL, it's a mocked socket with this send function. */ + int (*send_mock)(struct knot_xdp_socket *, const knot_xdp_msg_t[], uint32_t, uint32_t *); + + /*! The kernel has to be woken up by a syscall indication. */ + bool kernel_needs_wakeup; + + /*! The limit of frame size. */ + unsigned frame_limit; + + /*! Mapping of interface indices to VLAN tags. */ + uint16_t *vlan_map; + uint16_t vlan_map_max; +}; + +/*! + * \brief Set up BPF program and map for one XDP socket. + * + * \param if_name Name of the net iface (e.g. eth0). + * \param if_queue Network card queue id. + * \param load_bpf Insert BPF program into packet processing. + * \param out_iface Output: created interface context. + * + * \return KNOT_E* or -errno + */ +int kxsk_iface_new(const char *if_name, unsigned if_queue, knot_xdp_load_bpf_t load_bpf, + struct kxsk_iface **out_iface); + +/*! + * \brief Unload BPF maps for a socket. + * + * \note This keeps the loaded BPF program. We don't care. + * + * \param iface Interface context to be freed. + */ +void kxsk_iface_free(struct kxsk_iface *iface); + +/*! + * \brief Activate this AF_XDP socket through the BPF maps. + * + * \param iface Interface context. + * \param flags XDP filter configuration flags. + * \param udp_port UDP and/or TCP port to listen on if enabled via \a opts. + * \param quic_port QUIC/UDP port to listen on if enabled via \a opts. + * \param xsk Socket ctx. + * + * \return KNOT_E* or -errno + */ +int kxsk_socket_start(const struct kxsk_iface *iface, knot_xdp_filter_flag_t flags, + uint16_t udp_port, uint16_t quic_port, struct xsk_socket *xsk); + +/*! + * \brief Deactivate this AF_XDP socket through the BPF maps. + * + * \param iface Interface context. + */ +void kxsk_socket_stop(const struct kxsk_iface *iface); + +/*! @} */ diff --git a/src/libknot/xdp/eth.c b/src/libknot/xdp/eth.c new file mode 100644 index 0000000..608eb20 --- /dev/null +++ b/src/libknot/xdp/eth.c @@ -0,0 +1,312 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <bpf/libbpf.h> +#include <errno.h> +#include <ifaddrs.h> +#include <linux/ethtool.h> +#include <linux/if_link.h> +#include <linux/if_vlan.h> +#include <linux/sockios.h> +#include <net/if.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include "contrib/openbsd/strlcpy.h" +#include "contrib/sockaddr.h" +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/errcode.h" +#include "libknot/xdp/eth.h" + +_public_ +int knot_eth_queues(const char *devname) +{ + if (devname == NULL) { + return KNOT_EINVAL; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ethtool_channels ch = { + .cmd = ETHTOOL_GCHANNELS + }; + struct ifreq ifr = { + .ifr_data = (char *)&ch + }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + int ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + if (errno == EOPNOTSUPP) { + ret = 1; + } else { + ret = knot_map_errno(); + } + } else { + if (ch.combined_count == 0) { + ret = 1; + } else { + ret = ch.combined_count; + } + } + + close(fd); + return ret; +} + +_public_ +int knot_eth_rss(const char *devname, knot_eth_rss_conf_t **rss_conf) +{ + if (devname == NULL || rss_conf == NULL) { + return KNOT_EINVAL; + } + + struct ethtool_rxfh *ctx = NULL; + knot_eth_rss_conf_t *out = NULL; + int ret = KNOT_ERROR; + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ethtool_rxfh sizes = { + .cmd = ETHTOOL_GRSSH + }; + struct ifreq ifr = { + .ifr_data = (char *)&sizes + }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + ret = knot_map_errno(); + goto finish; + } + + const unsigned data_size = sizes.indir_size * sizeof(sizes.rss_config[0]) + + sizes.key_size; + + ctx = calloc(1, sizeof(*ctx) + data_size); + if (ctx == NULL) { + ret = KNOT_ENOMEM; + goto finish; + } + ctx->cmd = ETHTOOL_GRSSH; + ctx->indir_size = sizes.indir_size; + ctx->key_size = sizes.key_size; + ifr.ifr_data = (char *)ctx; + + ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + ret = knot_map_errno(); + goto finish; + } + + out = calloc(1, sizeof(*out) + data_size); + if (out == NULL) { + ret = KNOT_ENOMEM; + goto finish; + } + + out->table_size = sizes.indir_size; + out->key_size = sizes.key_size; + memcpy(out->data, ctx->rss_config, data_size); + out->mask = out->table_size - 1; +finish: + *rss_conf = out; + + free(ctx); + close(fd); + return ret; +} + +_public_ +int knot_eth_mtu(const char *devname) +{ + if (devname == NULL) { + return KNOT_EINVAL; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ifreq ifr = { 0 }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + int ret = ioctl(fd, SIOCGIFMTU, &ifr); + if (ret != 0) { + if (errno == EOPNOTSUPP) { + ret = KNOT_ENOTSUP; + } else { + ret = knot_map_errno(); + } + } else { + ret = ifr.ifr_mtu; + } + + close(fd); + return ret; +} + +_public_ +int knot_eth_name_from_addr(const struct sockaddr_storage *addr, char *out, + size_t out_len) +{ + if (addr == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct ifaddrs *ifaces = NULL; + if (getifaddrs(&ifaces) != 0) { + return -errno; + } + + size_t matches = 0; + char *match_name = NULL; + + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + const struct sockaddr_storage *ifss = (struct sockaddr_storage *)ifa->ifa_addr; + if (ifss == NULL) { // Observed on interfaces without any address. + continue; + } + + if ((ifss->ss_family == addr->ss_family && sockaddr_is_any(addr)) || + sockaddr_cmp(ifss, addr, true) == 0) { + matches++; + match_name = ifa->ifa_name; + } + } + + if (matches == 1) { + size_t len = strlcpy(out, match_name, out_len); + freeifaddrs(ifaces); + return (len >= out_len) ? KNOT_ESPACE : KNOT_EOK; + } + + freeifaddrs(ifaces); + return matches == 0 ? KNOT_EADDRNOTAVAIL : KNOT_ELIMIT; +} + +_public_ +int knot_eth_vlans(uint16_t *vlan_map[], uint16_t *vlan_map_max) +{ + if (vlan_map == NULL || vlan_map_max == NULL) { + return KNOT_EINVAL; + } + + struct ifaddrs *ifaces = NULL; + if (getifaddrs(&ifaces) != 0) { + return knot_map_errno(); + } + + unsigned map_size = 0; + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_PACKET) { + continue; + } + map_size++; + } + + uint16_t *map = calloc(sizeof(uint16_t), 1 + map_size); // Indexed from 1. + if (map == NULL) { + freeifaddrs(ifaces); + return KNOT_ENOMEM; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || ifa->ifa_addr->sa_family != AF_PACKET) { + continue; + } + + unsigned if_index = if_nametoindex(ifa->ifa_name); + if (if_index == 0) { + close(fd); + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + struct vlan_ioctl_args ifv = { + .cmd = GET_VLAN_REALDEV_NAME_CMD + }; + strlcpy(ifv.device1, ifa->ifa_name, sizeof(ifv.device1)); + + if (ioctl(fd, SIOCGIFVLAN, &ifv) >= 0) { + memset(&ifv, 0, sizeof(ifv)); + ifv.cmd = GET_VLAN_VID_CMD; + strlcpy(ifv.device1, ifa->ifa_name, sizeof(ifv.device1)); + + if (ioctl(fd, SIOCGIFVLAN, &ifv) < 0) { + close(fd); + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + map[if_index] = htobe16(ifv.u.VID); + } + } + + close(fd); + freeifaddrs(ifaces); + + *vlan_map = map; + *vlan_map_max = map_size; + + return KNOT_EOK; +} + +_public_ +knot_xdp_mode_t knot_eth_xdp_mode(int if_index) +{ +#if USE_LIBXDP + struct bpf_xdp_query_opts info = { .sz = sizeof(info) }; + int ret = bpf_xdp_query(if_index, 0, &info); +#else + struct xdp_link_info info; + int ret = bpf_get_link_xdp_info(if_index, &info, sizeof(info), 0); +#endif + if (ret != 0) { + return KNOT_XDP_MODE_NONE; + } + + switch (info.attach_mode) { + case XDP_ATTACHED_DRV: + case XDP_ATTACHED_HW: + return KNOT_XDP_MODE_FULL; + case XDP_ATTACHED_SKB: + return KNOT_XDP_MODE_EMUL; + default: + return KNOT_XDP_MODE_NONE; + } +} diff --git a/src/libknot/xdp/eth.h b/src/libknot/xdp/eth.h new file mode 100644 index 0000000..8c5b901 --- /dev/null +++ b/src/libknot/xdp/eth.h @@ -0,0 +1,111 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief Ethernet device info interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stddef.h> +#include <stdint.h> + +#define KNOT_XDP_MAX_MTU 1790 + +/*! + * \brief Get number of combined queues of a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * + * \retval < 0 KNOT_E* if error. + * \retval 1 Default no of queues if the dev does not support. + * \return > 0 Number of queues. + */ +int knot_eth_queues(const char *devname); + +/*! + * \brief Network card RSS configuration. + */ +typedef struct { + size_t table_size; /*!< Size of indirection table in four-bytes. */ + size_t key_size; /*!< Size of the RSS key in bytes. */ + uint32_t mask; /*!< Input mask for accessing the table. */ + uint32_t data[]; /*!< Serialized key and table. */ +} knot_eth_rss_conf_t; + +/*! + * \brief Get RSS configuration of a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * \param rss_conf Output RSS configuration. Must be freed explicitly. + * + * \return KNOT_E* + */ +int knot_eth_rss(const char *devname, knot_eth_rss_conf_t **rss_conf); + +/*! + * \brief Get value of MTU setup on a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * + * \retval < 0 KNOT_E* if error. + * \return >= 0 Interface MTU. + */ +int knot_eth_mtu(const char *devname); + +/*! + * \brief Get the corresponding network interface name for the address. + * + * \param addr Address of the interface. + * \param out Output buffer for the interface name. + * \param out_len Size of the output buffer. + * + * \return KNOT_E* + */ +int knot_eth_name_from_addr(const struct sockaddr_storage *addr, char *out, + size_t out_len); + +/*! + * \brief Get the mapping of interface index to VLAN tags. + * + * \param vlan_map Output array of the mappings. + * \param vlan_map_max Maximum interface index allowed. + * + * \return KNOT_E* + */ +int knot_eth_vlans(uint16_t *vlan_map[], uint16_t *vlan_map_max); + +typedef enum { + KNOT_XDP_MODE_NONE, /*!< XDP not available, BPF not loaded, or error. */ + KNOT_XDP_MODE_FULL, /*!< Full XDP support in driver or HW. */ + KNOT_XDP_MODE_EMUL, /*!< Emulated XDP support. */ +} knot_xdp_mode_t; + +/*! + * \brief Return the current XDP mode of a network interface. + * + * \param if_index Index of the interface, output from if_nametoindex(). + * + * \return Current XDP mode. + */ +knot_xdp_mode_t knot_eth_xdp_mode(int if_index); + +/*! @} */ diff --git a/src/libknot/xdp/msg.h b/src/libknot/xdp/msg.h new file mode 100644 index 0000000..f567921 --- /dev/null +++ b/src/libknot/xdp/msg.h @@ -0,0 +1,62 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP message description. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdint.h> +#include <linux/if_ether.h> +#include <netinet/in.h> +#include <sys/uio.h> + +/*! \brief Message flags. */ +typedef enum { + KNOT_XDP_MSG_IPV6 = (1 << 0), /*!< This packet is a IPv6 (IPv4 otherwise). */ + KNOT_XDP_MSG_TCP = (1 << 1), /*!< This packet is a TCP (UDP otherwise). */ + KNOT_XDP_MSG_SYN = (1 << 2), /*!< SYN flag set (TCP only). */ + KNOT_XDP_MSG_ACK = (1 << 3), /*!< ACK flag set (TCP only). */ + KNOT_XDP_MSG_FIN = (1 << 4), /*!< FIN flag set (TCP only). */ + KNOT_XDP_MSG_RST = (1 << 5), /*!< RST flag set (TCP only). */ + KNOT_XDP_MSG_MSS = (1 << 6), /*!< MSS option in TCP header (TCP only). */ + KNOT_XDP_MSG_WSC = (1 << 7), /*!< Window Scale option in TCP header. */ + KNOT_XDP_MSG_VLAN = (1 << 8), /*!< This packet will contain VLAN header. */ +} knot_xdp_msg_flag_t; + +/*! \brief Packet description with src & dst MAC & IP addrs + DNS payload. */ +typedef struct knot_xdp_msg { + struct sockaddr_in6 ip_from; + struct sockaddr_in6 ip_to; + uint8_t eth_from[ETH_ALEN]; + uint8_t eth_to[ETH_ALEN]; + knot_xdp_msg_flag_t flags; + struct iovec payload; + uint32_t seqno; + uint32_t ackno; + uint16_t mss; + uint16_t win; + uint8_t win_scale; + uint16_t vlan_tci; +} knot_xdp_msg_t; + +/*! @} */ diff --git a/src/libknot/xdp/msg_init.h b/src/libknot/xdp/msg_init.h new file mode 100644 index 0000000..8b96129 --- /dev/null +++ b/src/libknot/xdp/msg_init.h @@ -0,0 +1,74 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <stdbool.h> +#include <string.h> + +#include "libknot/xdp/msg.h" +#include "libknot/xdp/tcp.h" +#include "libdnssec/random.h" + +inline static bool empty_msg(const knot_xdp_msg_t *msg) +{ + const unsigned tcp_flags = KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK | + KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_RST; + + return (msg->payload.iov_len == 0 && !(msg->flags & tcp_flags)); +} + +inline static void msg_init_base(knot_xdp_msg_t *msg, knot_xdp_msg_flag_t flags) +{ + memset(msg, 0, sizeof(*msg)); + + msg->flags = flags; +} + +inline static void msg_init(knot_xdp_msg_t *msg, knot_xdp_msg_flag_t flags) +{ + msg_init_base(msg, flags); + + if (flags & KNOT_XDP_MSG_TCP) { + msg->ackno = 0; + msg->seqno = dnssec_random_uint32_t(); + if (flags & KNOT_XDP_MSG_SYN) { + msg->flags |= KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC; + } + } +} + +inline static void msg_init_reply(knot_xdp_msg_t *msg, const knot_xdp_msg_t *query) +{ + msg_init_base(msg, query->flags & (KNOT_XDP_MSG_IPV6 | KNOT_XDP_MSG_TCP | + KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC)); + + memcpy(msg->eth_from, query->eth_to, ETH_ALEN); + memcpy(msg->eth_to, query->eth_from, ETH_ALEN); + + memcpy(&msg->ip_from, &query->ip_to, sizeof(msg->ip_from)); + memcpy(&msg->ip_to, &query->ip_from, sizeof(msg->ip_to)); + + msg->vlan_tci = query->vlan_tci; + + if (msg->flags & KNOT_XDP_MSG_TCP) { + msg->ackno = knot_tcp_next_seqno(query); + msg->seqno = query->ackno; + if (msg->seqno == 0) { + msg->seqno = dnssec_random_uint32_t(); + } + } +} diff --git a/src/libknot/xdp/protocols.h b/src/libknot/xdp/protocols.h new file mode 100644 index 0000000..ab1cff4 --- /dev/null +++ b/src/libknot/xdp/protocols.h @@ -0,0 +1,446 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <assert.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <string.h> + +#include "libknot/endian.h" +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/msg.h" + +/* Don't fragment flag. */ +#define IP_DF 0x4000 + +#define HDR_8021Q_LEN 4; + +/* + * Following prot_read_*() functions do not check sanity of parsed packet. + * Broken packets have to be dropped by BPF filter prior getting here. + */ + +inline static void *prot_read_udp(void *data, uint16_t *src_port, uint16_t *dst_port) +{ + const struct udphdr *udp = data; + + *src_port = udp->source; + *dst_port = udp->dest; + + return data + sizeof(*udp); +} + +enum { + PROT_TCP_OPT_ENDOP = 0, + PROT_TCP_OPT_NOOP = 1, + PROT_TCP_OPT_MSS = 2, + PROT_TCP_OPT_WSC = 3, // window scale + + PROT_TCP_OPT_LEN_MSS = 4, + PROT_TCP_OPT_LEN_WSC = 3, +}; + +inline static void *prot_read_tcp(void *data, knot_xdp_msg_t *msg, uint16_t *src_port, uint16_t *dst_port) +{ + const struct tcphdr *tcp = data; + + msg->flags |= KNOT_XDP_MSG_TCP; + + if (tcp->syn) { + msg->flags |= KNOT_XDP_MSG_SYN; + } + if (tcp->ack) { + msg->flags |= KNOT_XDP_MSG_ACK; + } + if (tcp->fin) { + msg->flags |= KNOT_XDP_MSG_FIN; + } + if (tcp->rst) { + msg->flags |= KNOT_XDP_MSG_RST; + } + + msg->seqno = be32toh(tcp->seq); + msg->ackno = be32toh(tcp->ack_seq); + msg->win = be16toh(tcp->window); + + *src_port = tcp->source; + *dst_port = tcp->dest; + + uint8_t *opts = data + sizeof(*tcp), *hdr_end = data + tcp->doff * 4; + while (opts < hdr_end) { + if (opts[0] == PROT_TCP_OPT_ENDOP || opts[0] == PROT_TCP_OPT_NOOP) { + opts++; + continue; + } + + if (opts + 1 > hdr_end || opts + opts[1] > hdr_end) { + // Malformed option. + break; + } + + if (opts[0] == PROT_TCP_OPT_MSS && opts[1] == PROT_TCP_OPT_LEN_MSS) { + msg->flags |= KNOT_XDP_MSG_MSS; + memcpy(&msg->mss, &opts[2], sizeof(msg->mss)); + msg->mss = be16toh(msg->mss); + } + + if (opts[0] == PROT_TCP_OPT_WSC && opts[1] == PROT_TCP_OPT_LEN_WSC) { + msg->flags |= KNOT_XDP_MSG_WSC; + msg->win_scale = opts[2]; + } + + opts += opts[1]; + } + + return hdr_end; +} + +inline static void *prot_read_ipv4(void *data, knot_xdp_msg_t *msg, void **data_end) +{ + const struct iphdr *ip4 = data; + + // Conditions ensured by the BPF filter. + assert(ip4->version == 4); + assert(ip4->frag_off == 0 || ip4->frag_off == __constant_htons(IP_DF)); + // IPv4 header checksum is not verified! + + struct sockaddr_in *src = (struct sockaddr_in *)&msg->ip_from; + struct sockaddr_in *dst = (struct sockaddr_in *)&msg->ip_to; + memcpy(&src->sin_addr, &ip4->saddr, sizeof(src->sin_addr)); + memcpy(&dst->sin_addr, &ip4->daddr, sizeof(dst->sin_addr)); + src->sin_family = AF_INET; + dst->sin_family = AF_INET; + + *data_end = data + be16toh(ip4->tot_len); + data += ip4->ihl * 4; + + if (ip4->protocol == IPPROTO_TCP) { + return prot_read_tcp(data, msg, &src->sin_port, &dst->sin_port); + } else { + assert(ip4->protocol == IPPROTO_UDP); + return prot_read_udp(data, &src->sin_port, &dst->sin_port); + } +} + +inline static void *prot_read_ipv6(void *data, knot_xdp_msg_t *msg, void **data_end) +{ + const struct ipv6hdr *ip6 = data; + + msg->flags |= KNOT_XDP_MSG_IPV6; + + // Conditions ensured by the BPF filter. + assert(ip6->version == 6); + + struct sockaddr_in6 *src = (struct sockaddr_in6 *)&msg->ip_from; + struct sockaddr_in6 *dst = (struct sockaddr_in6 *)&msg->ip_to; + memcpy(&src->sin6_addr, &ip6->saddr, sizeof(src->sin6_addr)); + memcpy(&dst->sin6_addr, &ip6->daddr, sizeof(dst->sin6_addr)); + src->sin6_family = AF_INET6; + dst->sin6_family = AF_INET6; + src->sin6_flowinfo = 0; + dst->sin6_flowinfo = 0; + // Scope ID is ignored. + + data += sizeof(*ip6); + *data_end = data + be16toh(ip6->payload_len); + + if (ip6->nexthdr == IPPROTO_TCP) { + return prot_read_tcp(data, msg, &src->sin6_port, &dst->sin6_port); + } else { + assert(ip6->nexthdr == IPPROTO_UDP); + return prot_read_udp(data, &src->sin6_port, &dst->sin6_port); + } +} + +inline static void *prot_read_eth(void *data, knot_xdp_msg_t *msg, void **data_end, + const uint16_t *vlan_map, unsigned vlan_map_max) +{ + const struct ethhdr *eth = data; + knot_xdp_info_t *info = data - KNOT_XDP_PKT_ALIGNMENT - sizeof(*info); + + memcpy(msg->eth_from, eth->h_source, ETH_ALEN); + memcpy(msg->eth_to, eth->h_dest, ETH_ALEN); + msg->flags = 0; + + if (eth->h_proto == __constant_htons(ETH_P_8021Q)) { + if (info->out_if_index > 0 && info->out_if_index <= vlan_map_max) { + assert(vlan_map); + msg->vlan_tci = vlan_map[info->out_if_index]; + } else { + memcpy(&msg->vlan_tci, data + sizeof(*eth), sizeof(msg->vlan_tci)); + } + data += HDR_8021Q_LEN; + eth = data; + } + + data += sizeof(*eth); + + if (eth->h_proto == __constant_htons(ETH_P_IPV6)) { + return prot_read_ipv6(data, msg, data_end); + } else { + assert(eth->h_proto == __constant_htons(ETH_P_IP)); + return prot_read_ipv4(data, msg, data_end); + } +} + +inline static size_t prot_write_hdrs_len(const knot_xdp_msg_t *msg) +{ + size_t res = sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr); + + if (msg->vlan_tci != 0 || msg->flags & KNOT_XDP_MSG_VLAN) { + res += HDR_8021Q_LEN; + } + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + res += sizeof(struct ipv6hdr) - sizeof(struct iphdr); + } + + if (msg->flags & KNOT_XDP_MSG_TCP) { + res += sizeof(struct tcphdr) - sizeof(struct udphdr); + + if (msg->flags & KNOT_XDP_MSG_MSS) { + res += PROT_TCP_OPT_LEN_MSS; + } + if (msg->flags & KNOT_XDP_MSG_WSC) { + res += PROT_TCP_OPT_LEN_WSC + 1; // 1 == align + } + } + + return res; +} + +/* Checksum endianness implementation notes for ipv4_checksum() and checksum(). + * + * The basis for checksum is addition on big-endian 16-bit words, with bit 16 carrying + * over to bit 0. That can be viewed as first byte carrying to the second and the + * second one carrying back to the first one, i.e. a symmetrical situation. + * Therefore the result is the same even when arithmetics is done on little-endian (!) + */ + +inline static void checksum(uint32_t *result, const void *_data, uint32_t _data_len) +{ + assert(!(_data_len & 1)); + const uint16_t *data = _data; + uint32_t len = _data_len / 2; + while (len-- > 0) { + *result += *data++; + } +} + +inline static void checksum_uint16(uint32_t *result, uint16_t x) +{ + checksum(result, &x, sizeof(x)); +} + +inline static void checksum_payload(uint32_t *result, void *payload, size_t pay_len) +{ + if (pay_len & 1) { + ((uint8_t *)payload)[pay_len++] = 0; + } + checksum(result, payload, pay_len); +} + +inline static uint16_t checksum_finish(uint32_t result, bool nonzero) +{ + while (result > 0xffff) { + result = (result & 0xffff) + (result >> 16); + } + if (!nonzero || result != 0xffff) { + result = ~result; + } + return result; +} + +inline static void prot_write_udp(void *data, const knot_xdp_msg_t *msg, void *data_end, + uint16_t src_port, uint16_t dst_port, uint32_t chksum) +{ + struct udphdr *udp = data; + + udp->len = htobe16(data_end - data); + udp->source = src_port; + udp->dest = dst_port; + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + udp->check = 0; + checksum(&chksum, &udp->len, sizeof(udp->len)); + checksum_uint16(&chksum, htobe16(IPPROTO_UDP)); + checksum_payload(&chksum, data, data_end - data); + udp->check = checksum_finish(chksum, true); + } else { + udp->check = 0; // UDP over IPv4 doesn't require checksum. + } + + assert(data + sizeof(*udp) == msg->payload.iov_base); +} + +inline static void prot_write_tcp(void *data, const knot_xdp_msg_t *msg, void *data_end, + uint16_t src_port, uint16_t dst_port, uint32_t chksum, + uint16_t mss) +{ + struct tcphdr *tcp = data; + + tcp->source = src_port; + tcp->dest = dst_port; + tcp->seq = htobe32(msg->seqno); + tcp->ack_seq = htobe32(msg->ackno); + tcp->window = htobe16(msg->win); + tcp->check = 0; // Temporarily initialize before checksum calculation. + + tcp->syn = ((msg->flags & KNOT_XDP_MSG_SYN) ? 1 : 0); + tcp->ack = ((msg->flags & KNOT_XDP_MSG_ACK) ? 1 : 0); + tcp->fin = ((msg->flags & KNOT_XDP_MSG_FIN) ? 1 : 0); + tcp->rst = ((msg->flags & KNOT_XDP_MSG_RST) ? 1 : 0); + + uint8_t *hdr_end = data + sizeof(*tcp); + if (msg->flags & KNOT_XDP_MSG_WSC) { + hdr_end[0] = PROT_TCP_OPT_WSC; + hdr_end[1] = PROT_TCP_OPT_LEN_WSC; + hdr_end[2] = msg->win_scale; + hdr_end += PROT_TCP_OPT_LEN_WSC; + *hdr_end++ = PROT_TCP_OPT_NOOP; // align + } + if (msg->flags & KNOT_XDP_MSG_MSS) { + mss = htobe16(mss); + hdr_end[0] = PROT_TCP_OPT_MSS; + hdr_end[1] = PROT_TCP_OPT_LEN_MSS; + memcpy(&hdr_end[2], &mss, sizeof(mss)); + hdr_end += PROT_TCP_OPT_LEN_MSS; + } + + tcp->psh = ((data_end - (void *)hdr_end > 0) ? 1 : 0); + tcp->doff = (hdr_end - (uint8_t *)tcp) / 4; + assert((hdr_end - (uint8_t *)tcp) % 4 == 0); + + checksum_uint16(&chksum, htobe16(IPPROTO_TCP)); + checksum_uint16(&chksum, htobe16(data_end - data)); + checksum_payload(&chksum, data, data_end - data); + tcp->check = checksum_finish(chksum, false); + + assert(hdr_end == msg->payload.iov_base); +} + +inline static uint16_t from32to16(uint32_t sum) +{ + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return sum; +} + +inline static uint16_t ipv4_checksum(const uint16_t *ipv4_hdr) +{ + uint32_t sum32 = 0; + for (int i = 0; i < 10; ++i) { + if (i != 5) { + sum32 += ipv4_hdr[i]; + } + } + return ~from32to16(sum32); +} + +inline static void prot_write_ipv4(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct iphdr *ip4 = data; + + ip4->version = 4; + ip4->ihl = sizeof(*ip4) / 4; + ip4->tos = 0; + ip4->tot_len = htobe16(data_end - data); + ip4->id = 0; + ip4->frag_off = 0; + ip4->ttl = IPDEFTTL; + ip4->protocol = ((msg->flags & KNOT_XDP_MSG_TCP) ? IPPROTO_TCP : IPPROTO_UDP); + + const struct sockaddr_in *src = (const struct sockaddr_in *)&msg->ip_from; + const struct sockaddr_in *dst = (const struct sockaddr_in *)&msg->ip_to; + memcpy(&ip4->saddr, &src->sin_addr, sizeof(src->sin_addr)); + memcpy(&ip4->daddr, &dst->sin_addr, sizeof(dst->sin_addr)); + + ip4->check = ipv4_checksum(data); + + data += sizeof(*ip4); + + if (msg->flags & KNOT_XDP_MSG_TCP) { + uint32_t chk = 0; + checksum(&chk, &src->sin_addr, sizeof(src->sin_addr)); + checksum(&chk, &dst->sin_addr, sizeof(dst->sin_addr)); + + prot_write_tcp(data, msg, data_end, src->sin_port, dst->sin_port, chk, tcp_mss); + } else { + prot_write_udp(data, msg, data_end, src->sin_port, dst->sin_port, 0); // IPv4/UDP requires no checksum + } +} + +inline static void prot_write_ipv6(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct ipv6hdr *ip6 = data; + + ip6->version = 6; + ip6->priority = 0; + ip6->payload_len = htobe16(data_end - data - sizeof(*ip6)); + ip6->nexthdr = ((msg->flags & KNOT_XDP_MSG_TCP) ? IPPROTO_TCP : IPPROTO_UDP); + ip6->hop_limit = IPDEFTTL; + + memset(ip6->flow_lbl, 0, sizeof(ip6->flow_lbl)); + + const struct sockaddr_in6 *src = (const struct sockaddr_in6 *)&msg->ip_from; + const struct sockaddr_in6 *dst = (const struct sockaddr_in6 *)&msg->ip_to; + memcpy(&ip6->saddr, &src->sin6_addr, sizeof(src->sin6_addr)); + memcpy(&ip6->daddr, &dst->sin6_addr, sizeof(dst->sin6_addr)); + + data += sizeof(*ip6); + + uint32_t chk = 0; + checksum(&chk, &src->sin6_addr, sizeof(src->sin6_addr)); + checksum(&chk, &dst->sin6_addr, sizeof(dst->sin6_addr)); + + if (msg->flags & KNOT_XDP_MSG_TCP) { + prot_write_tcp(data, msg, data_end, src->sin6_port, dst->sin6_port, chk, tcp_mss); + } else { + prot_write_udp(data, msg, data_end, src->sin6_port, dst->sin6_port, chk); + } +} + +inline static void prot_write_eth(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct ethhdr *eth = data; + + memcpy(eth->h_source, msg->eth_from, ETH_ALEN); + memcpy(eth->h_dest, msg->eth_to, ETH_ALEN); + + if (msg->vlan_tci != 0) { + eth->h_proto = __constant_htons(ETH_P_8021Q); + memcpy(data + sizeof(*eth), &msg->vlan_tci, sizeof(msg->vlan_tci)); + data += HDR_8021Q_LEN; + eth = data; + } + + data += sizeof(*eth); + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + eth->h_proto = __constant_htons(ETH_P_IPV6); + prot_write_ipv6(data, msg, data_end, tcp_mss); + } else { + eth->h_proto = __constant_htons(ETH_P_IP); + prot_write_ipv4(data, msg, data_end, tcp_mss); + } +} diff --git a/src/libknot/xdp/quic.c b/src/libknot/xdp/quic.c new file mode 100644 index 0000000..f12a47c --- /dev/null +++ b/src/libknot/xdp/quic.c @@ -0,0 +1,1028 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <gnutls/gnutls.h> +#include <gnutls/crypto.h> +#include <gnutls/x509.h> +#include <ngtcp2/ngtcp2.h> +#include <ngtcp2/ngtcp2_crypto.h> +#include <ngtcp2/ngtcp2_crypto_gnutls.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +#include "libknot/xdp/quic.h" + +#include "contrib/macros.h" +#include "contrib/sockaddr.h" +#include "contrib/ucw/lists.h" +#include "libknot/endian.h" +#include "libdnssec/error.h" +#include "libdnssec/random.h" +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/error.h" +#include "libknot/wire.h" + +#define SERVER_DEFAULT_SCIDLEN 18 + +#define QUIC_DEFAULT_VERSION "-VERS-ALL:+VERS-TLS1.3" +#define QUIC_DEFAULT_CIPHERS "-CIPHER-ALL:+AES-128-GCM:+AES-256-GCM:+CHACHA20-POLY1305:+AES-128-CCM" +#define QUIC_DEFAULT_GROUPS "-GROUP-ALL:+GROUP-SECP256R1:+GROUP-X25519:+GROUP-SECP384R1:+GROUP-SECP521R1" +#define QUIC_PRIORITIES "%DISABLE_TLS13_COMPAT_MODE:NORMAL:"QUIC_DEFAULT_VERSION":"QUIC_DEFAULT_CIPHERS":"QUIC_DEFAULT_GROUPS + +#define XQUIC_SEND_VERSION_NEGOTIATION NGTCP2_ERR_VERSION_NEGOTIATION +#define XQUIC_SEND_RETRY NGTCP2_ERR_RETRY +#define XQUIC_SEND_STATELESS_RESET (-NGTCP2_STATELESS_RESET_TOKENLEN) + +#define TLS_CALLBACK_ERR (-1) + +typedef struct knot_quic_creds { + gnutls_certificate_credentials_t tls_cert; + gnutls_anti_replay_t tls_anti_replay; + gnutls_datum_t tls_ticket_key; + bool is_clone; +} knot_xquic_creds_t; + +typedef struct knot_quic_session { + node_t n; + gnutls_datum_t tls_session; + ngtcp2_transport_params quic_params; +} knot_xquic_session_t; + +static unsigned addr_len(const struct sockaddr_in6 *ss) +{ + return (ss->sin6_family == AF_INET6 ? + sizeof(struct sockaddr_in6) : sizeof(struct sockaddr)); +} + +_public_ +struct knot_quic_session *knot_xquic_session_save(knot_xquic_conn_t *conn) +{ + const ngtcp2_transport_params *tmp = ngtcp2_conn_get_remote_transport_params(conn->conn); + if (tmp == NULL) { + return NULL; + } + + knot_xquic_session_t *session = calloc(1, sizeof(*session)); + if (session == NULL) { + return NULL; + } + + int ret = gnutls_session_get_data2(conn->tls_session, &session->tls_session); + if (ret != GNUTLS_E_SUCCESS) { + free(session); + return NULL; + } + + memcpy(&session->quic_params, tmp, sizeof(session->quic_params)); + + return session; +} + +_public_ +int knot_xquic_session_load(knot_xquic_conn_t *conn, struct knot_quic_session *session) +{ + if (session == NULL) { + return KNOT_EINVAL; + } + + int ret = KNOT_EOK; + if (conn == NULL) { + goto session_free; + } + + ret = gnutls_session_set_data(conn->tls_session, session->tls_session.data, session->tls_session.size); + if (ret != KNOT_EOK) { + goto session_free; + } + + ngtcp2_conn_set_early_remote_transport_params(conn->conn, &session->quic_params); + +session_free: + gnutls_free(session->tls_session.data); + free(session); + return ret; +} + +static int tls_anti_replay_db_add_func(void *dbf, time_t exp_time, + const gnutls_datum_t *key, + const gnutls_datum_t *data) +{ + return 0; +} + +static void tls_session_ticket_key_free(gnutls_datum_t *ticket) +{ + gnutls_memset(ticket->data, 0, ticket->size); + gnutls_free(ticket->data); +} + +static int self_signed_cert(gnutls_certificate_credentials_t tls_cert) +{ + gnutls_x509_privkey_t privkey = NULL; + gnutls_x509_crt_t cert = NULL; + char *hostname = sockaddr_hostname(); + if (hostname == NULL) { + return -ENOMEM; + } + + int ret = gnutls_x509_privkey_init(&privkey); + if (ret < 0) { + free(hostname); + return ret; + } + + uint8_t serial[16]; + gnutls_rnd(GNUTLS_RND_NONCE, serial, sizeof(serial)); + /* clear the left-most bit to avoid signedness confusion: */ + serial[0] &= 0x8f; + +#define CHK(cmd) \ + ret = (cmd); \ + if (ret < 0) { goto finish; } + +#define now_years(years) (time(NULL) + 365 * 24 * 3600 * (years)) + + CHK(gnutls_x509_privkey_generate(privkey, GNUTLS_PK_ECDSA, GNUTLS_CURVE_TO_BITS(GNUTLS_ECC_CURVE_SECP256R1), 0)); + + CHK(gnutls_x509_crt_init(&cert)); + //CHK(gnutls_x509_crt_set_ca_status(cert, 0)); // TODO needed ? + CHK(gnutls_x509_crt_set_activation_time(cert, now_years(-1))); + CHK(gnutls_x509_crt_set_expiration_time(cert, now_years(20))); + CHK(gnutls_x509_crt_set_dn(cert, "CN=DoQ Self-Signed Server Certificate", NULL)); + CHK(gnutls_x509_crt_set_key(cert, privkey)); + + CHK(gnutls_x509_crt_set_serial(cert, serial, sizeof(serial))); + CHK(gnutls_x509_crt_set_subject_alt_name(cert, GNUTLS_SAN_DNSNAME, hostname, strlen(hostname), GNUTLS_FSAN_SET)); + CHK(gnutls_x509_crt_set_version(cert, 3)); + CHK(gnutls_x509_crt_sign2(cert, cert, privkey, GNUTLS_DIG_SHA256, 0)); + + ret = gnutls_certificate_set_x509_key(tls_cert, &cert, 1, privkey); + +finish: + free(hostname); + gnutls_x509_privkey_deinit(privkey); + gnutls_x509_crt_deinit(cert); + + return ret; +} + +_public_ +struct knot_quic_creds *knot_xquic_init_creds(bool server, const char *tls_cert, + const char *tls_key) +{ + knot_xquic_creds_t *creds = calloc(1, sizeof(*creds)); + if (creds == NULL) { + return NULL; + } + + int ret = gnutls_anti_replay_init(&creds->tls_anti_replay); + if (ret != GNUTLS_E_SUCCESS) { + goto fail; + } + gnutls_anti_replay_set_add_function(creds->tls_anti_replay, tls_anti_replay_db_add_func); + gnutls_anti_replay_set_ptr(creds->tls_anti_replay, NULL); + + ret = gnutls_certificate_allocate_credentials(&creds->tls_cert); + if (ret != GNUTLS_E_SUCCESS) { + gnutls_anti_replay_deinit(creds->tls_anti_replay); + goto fail; + } + + ret = gnutls_certificate_set_x509_system_trust(creds->tls_cert); + if (ret < 0) { + goto fail2; + } + + if ((bool)(tls_cert == NULL) != (bool)(tls_key == NULL) || + (tls_cert != NULL && !server)) { + goto fail2; + } + if (tls_cert != NULL) { + ret = gnutls_certificate_set_x509_key_file(creds->tls_cert, tls_cert, tls_key, GNUTLS_X509_FMT_PEM); + } else if (server) { + ret = self_signed_cert(creds->tls_cert); + } + if (ret < 0) { + goto fail2; + } + + ret = gnutls_session_ticket_key_generate(&creds->tls_ticket_key); + if (ret != GNUTLS_E_SUCCESS) { + goto fail2; + } + + return creds; + +fail: + free(creds); + return NULL; +fail2: + knot_xquic_free_creds(creds); + return NULL; +} + +_public_ +void knot_xquic_free_creds(struct knot_quic_creds *creds) +{ + if (creds == NULL) { + return; + } + + if (!creds->is_clone) { + gnutls_certificate_free_credentials(creds->tls_cert); + if (creds->tls_ticket_key.data != NULL) { + tls_session_ticket_key_free(&creds->tls_ticket_key); + } + } + gnutls_anti_replay_deinit(creds->tls_anti_replay); + free(creds); +} + +#define ALPN "\03""doq" +#define ALPN_TMP1 "\07""doq-i11" +#define ALPN_TMP2 "\07""doq-i03" + +static int tls_client_hello_cb(gnutls_session_t session, unsigned int htype, + unsigned when, unsigned int incoming, + const gnutls_datum_t *msg) +{ + assert(htype == GNUTLS_HANDSHAKE_CLIENT_HELLO); + assert(when == GNUTLS_HOOK_POST); + + if (!incoming) { + return 0; + } + + gnutls_datum_t alpn; + int ret = gnutls_alpn_get_selected_protocol(session, &alpn); + if (ret != 0) { + return ret; + } + + const char *dq = (const char *)&ALPN[1]; + if (((unsigned int)ALPN[0] != alpn.size || + memcmp(dq, alpn.data, alpn.size) != 0) && + ((unsigned int)ALPN_TMP1[0] != alpn.size || + memcmp((const char *)&ALPN_TMP1[1], alpn.data, alpn.size) != 0) && + ((unsigned int)ALPN_TMP1[0] != alpn.size || + memcmp((const char *)&ALPN_TMP2[1], alpn.data, alpn.size) != 0)) { + return TLS_CALLBACK_ERR; + } + + return 0; +} + +static ngtcp2_conn *get_conn(ngtcp2_crypto_conn_ref *conn_ref) +{ + return ((knot_xquic_conn_t *)conn_ref->user_data)->conn; +} + +static int tls_init_conn_session(knot_xquic_conn_t *conn, bool server) +{ + if (gnutls_init(&conn->tls_session, (server ? GNUTLS_SERVER : GNUTLS_CLIENT) | + GNUTLS_ENABLE_EARLY_DATA | GNUTLS_NO_AUTO_SEND_TICKET | + GNUTLS_NO_END_OF_EARLY_DATA) != GNUTLS_E_SUCCESS) { + return TLS_CALLBACK_ERR; + } + + if (gnutls_priority_set_direct(conn->tls_session, QUIC_PRIORITIES, + NULL) != GNUTLS_E_SUCCESS) { + return TLS_CALLBACK_ERR; + } + + if (server && gnutls_session_ticket_enable_server(conn->tls_session, + &conn->xquic_table->creds->tls_ticket_key) != GNUTLS_E_SUCCESS) { + return TLS_CALLBACK_ERR; + } + + gnutls_handshake_set_hook_function(conn->tls_session, + GNUTLS_HANDSHAKE_CLIENT_HELLO, + GNUTLS_HOOK_POST, tls_client_hello_cb); + int ret = ngtcp2_crypto_gnutls_configure_server_session(conn->tls_session); + if (ret != 0) { + return TLS_CALLBACK_ERR; + } + + gnutls_record_set_max_early_data_size(conn->tls_session, 0xffffffffu); + + conn->conn_ref = (nc_conn_ref_placeholder_t) { + .get_conn = get_conn, + .user_data = conn + }; + + _Static_assert(sizeof(nc_conn_ref_placeholder_t) == sizeof(ngtcp2_crypto_conn_ref), "invalid placeholder for conn_ref"); + gnutls_session_set_ptr(conn->tls_session, &conn->conn_ref); + + if (server) { + gnutls_anti_replay_enable(conn->tls_session, conn->xquic_table->creds->tls_anti_replay); + + } + if (gnutls_credentials_set(conn->tls_session, GNUTLS_CRD_CERTIFICATE, + conn->xquic_table->creds->tls_cert) != GNUTLS_E_SUCCESS) { + return TLS_CALLBACK_ERR; + } + + + gnutls_datum_t alpn[3] = { + { + .data = (uint8_t *)(&ALPN[1]), + .size = ALPN[0], + }, + { + .data = (uint8_t *)(&ALPN_TMP1[1]), + .size = ALPN_TMP1[0], + }, + { + .data = (uint8_t *)(&ALPN_TMP2[1]), + .size = ALPN_TMP2[0], + } + }; + gnutls_alpn_set_protocols(conn->tls_session, alpn, 3, 0); + + ngtcp2_conn_set_tls_native_handle(conn->conn, conn->tls_session); + + return KNOT_EOK; +} + +static uint64_t get_timestamp(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + assert(0); + } + + return (uint64_t)ts.tv_sec * NGTCP2_SECONDS + (uint64_t)ts.tv_nsec; +} + +uint64_t xquic_conn_get_timeout(knot_xquic_conn_t *conn) +{ + // This effectively obtains the locally configured conn timeout. + // It would be possible to obey negotitated idle timeout by employing remote params, + // but this would differ per-connection and the whole idea of maintaining + // to-be-timeouted connections in simple linear list requires that + // the idle timeout is homogeneous among conns. + // Anyway, we also violate RFC9000/10.1 (Probe Timeout) for the same reason. + // TODO for the future: refactor conn table to use some tree/heap + // for to-be-timeouted conns, and use ngtcp2_conn_get_expiry() and + // ngtcp2_conn_handle_expiry() appropriately. + const ngtcp2_transport_params *params = ngtcp2_conn_get_local_transport_params(conn->conn); + + return conn->last_ts + params->max_idle_timeout; +} + +bool xquic_conn_timeout(knot_xquic_conn_t *conn, uint64_t *now) +{ + if (*now == 0) { + *now = get_timestamp(); + } + return *now > xquic_conn_get_timeout(conn); +} + +_public_ +uint32_t knot_xquic_conn_rtt(knot_xquic_conn_t *conn) +{ + ngtcp2_conn_stat stat = { 0 }; + ngtcp2_conn_get_conn_stat(conn->conn, &stat); + return stat.smoothed_rtt / 1000; // nanosec --> usec +} + +static void knot_quic_rand_cb(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) +{ + (void)rand_ctx; + dnssec_random_buffer(dest, destlen); +} + +static void init_random_cid(ngtcp2_cid *cid, size_t len) +{ + if (len == 0) { + len = SERVER_DEFAULT_SCIDLEN; + } + + if (dnssec_random_buffer(cid->data, len) != DNSSEC_EOK) { + cid->datalen = 0; + } else { + cid->datalen = len; + } +} + +static bool init_unique_cid(ngtcp2_cid *cid, size_t len, knot_xquic_table_t *table) +{ + do { + if (init_random_cid(cid, len), cid->datalen == 0) { + return false; + } + } while (xquic_table_lookup(cid, table) != NULL); + return true; +} + +static int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, + uint8_t *token, size_t cidlen, + void *user_data) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + if (!init_unique_cid(cid, cidlen, ctx->xquic_table)) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + knot_xquic_cid_t **addto = xquic_table_insert(ctx, cid, ctx->xquic_table); + (void)addto; + + if (token != NULL && + ngtcp2_crypto_generate_stateless_reset_token( + token, (uint8_t *)ctx->xquic_table->hash_secret, + sizeof(ctx->xquic_table->hash_secret), cid) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} + +static int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid, + void *user_data) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + knot_xquic_cid_t **torem = xquic_table_lookup2(cid, ctx->xquic_table); + if (torem != NULL) { + assert((*torem)->conn == ctx); + xquic_table_rem2(torem, ctx->xquic_table); + } + + return 0; +} + +static int handshake_completed_cb(ngtcp2_conn *conn, void *user_data) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + assert(!ctx->handshake_done); + ctx->handshake_done = true; + + if (!ngtcp2_conn_is_server(conn)) { + return 0; + } + + gnutls_datum_t alpn; + if (gnutls_alpn_get_selected_protocol(ctx->tls_session, &alpn) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + char alpn_str[alpn.size + 1]; + alpn_str[alpn.size] = '\0'; + memcpy(alpn_str, alpn.data, alpn.size); + + if (gnutls_session_ticket_send(ctx->tls_session, 1, 0) != GNUTLS_E_SUCCESS) { + return TLS_CALLBACK_ERR; + } + + uint8_t token[NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN]; + ngtcp2_path path = *ngtcp2_conn_get_path(ctx->conn); + uint64_t ts = get_timestamp(); + ngtcp2_ssize tokenlen = ngtcp2_crypto_generate_regular_token(token, + (uint8_t *)ctx->xquic_table->hash_secret, + sizeof(ctx->xquic_table->hash_secret), + path.remote.addr, path.remote.addrlen, ts); + if (tokenlen < 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + if (ngtcp2_conn_submit_new_token(ctx->conn, token, tokenlen) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} + +static int recv_stream_data(ngtcp2_conn *conn, uint32_t flags, + int64_t stream_id, uint64_t offset, + const uint8_t *data, size_t datalen, + void *user_data, void *stream_user_data) +{ + (void)(stream_user_data); // always NULL + (void)(offset); // QUIC shall ensure that data arrive in-order + + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + int ret = knot_xquic_stream_recv_data(ctx, stream_id, data, datalen, (flags & NGTCP2_STREAM_DATA_FLAG_FIN)); + + return ret == KNOT_EOK ? 0 : NGTCP2_ERR_CALLBACK_FAILURE; +} + +static int acked_stream_data_offset_cb(ngtcp2_conn *conn, int64_t stream_id, + uint64_t offset, uint64_t datalen, + void *user_data, void *stream_user_data) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + + bool keep = !ngtcp2_conn_is_server(conn); // kxdpgun: await incomming reply after query sent&acked + + knot_xquic_stream_ack_data(ctx, stream_id, offset + datalen, keep); + + return 0; +} + +static int stream_closed(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id, + uint64_t app_error_code, void *user_data, void *stream_user_data) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + // NOTE possible error is stored in (flags & NGTCP2_STREAM_CLOSE_FLAG_APP_ERROR_CODE_SET) + + bool keep = !ngtcp2_conn_is_server(conn); // kxdpgun: process incomming reply after recvd&closed + if (!keep) { + xquic_stream_free(ctx, stream_id); + } + return 0; +} + +static int recv_stateless_rst(ngtcp2_conn *conn, const ngtcp2_pkt_stateless_reset *sr, void *user_data) +{ + // NOTE server can't receive stateless resets, only client + + // ngtcp2 verified stateless reset token already + (void)(sr); + + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + assert(ctx->conn == conn); + + knot_xquic_table_rem(ctx, ctx->xquic_table); + + return 0; +} + +static int recv_stream_rst(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size, + uint64_t app_error_code, void *user_data, void *stream_user_data) +{ + (void)final_size; + return stream_closed(conn, NGTCP2_STREAM_CLOSE_FLAG_APP_ERROR_CODE_SET, + stream_id, app_error_code, user_data, stream_user_data); +} + +static void user_printf(void *user_data, const char *format, ...) +{ + knot_xquic_conn_t *ctx = (knot_xquic_conn_t *)user_data; + if (ctx->xquic_table->log_cb != NULL) { + char buf[256]; + va_list args; + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + ctx->xquic_table->log_cb(buf); + } +} + +static int conn_new(ngtcp2_conn **pconn, const ngtcp2_path *path, const ngtcp2_cid *scid, + const ngtcp2_cid *dcid, const ngtcp2_cid *odcid, uint32_t version, + uint64_t now, size_t udp_pl, uint64_t idle_timeout_ns, + void *user_data, bool server, bool retry_sent) +{ + // I. CALLBACKS + const ngtcp2_callbacks callbacks = { + ngtcp2_crypto_client_initial_cb, + ngtcp2_crypto_recv_client_initial_cb, + ngtcp2_crypto_recv_crypto_data_cb, + handshake_completed_cb, + NULL, // recv_version_negotiation not needed on server, nor kxdpgun + ngtcp2_crypto_encrypt_cb, + ngtcp2_crypto_decrypt_cb, + ngtcp2_crypto_hp_mask_cb, + recv_stream_data, + acked_stream_data_offset_cb, + NULL, // stream_opened + stream_closed, + recv_stateless_rst, + ngtcp2_crypto_recv_retry_cb, + NULL, // extend_max_streams_bidi + NULL, // extend_max_streams_uni + knot_quic_rand_cb, + get_new_connection_id, + remove_connection_id, + ngtcp2_crypto_update_key_cb, + NULL, // path_validation, + NULL, // select_preferred_addr + recv_stream_rst, + NULL, // extend_max_remote_streams_bidi, might be useful to some allocation optimizations? + NULL, // extend_max_remote_streams_uni + NULL, // extend_max_stream_data, + NULL, // dcid_status + NULL, // handshake_confirmed + NULL, // recv_new_token + ngtcp2_crypto_delete_crypto_aead_ctx_cb, + ngtcp2_crypto_delete_crypto_cipher_ctx_cb, + NULL, // recv_datagram + NULL, // ack_datagram + NULL, // lost_datagram + ngtcp2_crypto_get_path_challenge_data_cb, + NULL, // stream_stop_sending + ngtcp2_crypto_version_negotiation_cb, + NULL, // recv_rx_key + NULL // recv_tx_key + }; + + // II. SETTINGS + ngtcp2_settings settings; + ngtcp2_settings_default(&settings); + settings.initial_ts = now; + settings.log_printf = user_printf; + settings.max_tx_udp_payload_size = udp_pl; + settings.qlog.odcid = *odcid; + settings.handshake_timeout = idle_timeout_ns; // NOTE setting handshake timeout to idle_timeout for simplicity + settings.no_pmtud = true; + + // III. PARAMS + ngtcp2_transport_params params; + ngtcp2_transport_params_default(¶ms); + + params.initial_max_data = 786432; + params.initial_max_stream_data_bidi_local = 524288; + params.initial_max_stream_data_bidi_remote = 524288; + params.initial_max_stream_data_uni = 524288; + + // params.initial_max_stream_data_bidi_local = config.max_stream_data_bidi_local; + // params.initial_max_stream_data_bidi_remote = config.max_stream_data_bidi_remote; + // params.initial_max_stream_data_uni = config.max_stream_data_uni; + // params.initial_max_data = config.max_data; + params.initial_max_streams_bidi = 100; + params.initial_max_streams_uni = 3; + params.max_idle_timeout = idle_timeout_ns; + // params.stateless_reset_token_present = 1; + // params.active_connection_id_limit = 7; + if (odcid) { + params.original_dcid = *odcid; + } else { + params.original_dcid = *scid; + } + if (retry_sent) { + params.retry_scid_present = 1; + params.retry_scid = *scid; + } + if (dnssec_random_buffer(params.stateless_reset_token, NGTCP2_STATELESS_RESET_TOKENLEN) != DNSSEC_EOK) { + return KNOT_ERROR; + } + + if (server) { + return ngtcp2_conn_server_new(pconn, dcid, scid, path, version, &callbacks, &settings, ¶ms, NULL, user_data); + } else { + return ngtcp2_conn_client_new(pconn, dcid, scid, path, version, &callbacks, &settings, ¶ms, NULL, user_data); + } +} + +_public_ +int knot_xquic_client(knot_xquic_table_t *table, struct sockaddr_in6 *dest, + struct sockaddr_in6 *via, knot_xquic_conn_t **out_conn) +{ + ngtcp2_cid scid = { 0 }, dcid = { 0 }; + uint64_t now = get_timestamp(); + + init_random_cid(&scid, 0); + init_random_cid(&dcid, 0); + + knot_xquic_conn_t *xconn = xquic_table_add(NULL, &dcid, table); + if (xconn == NULL) { + return ENOMEM; + } + xquic_conn_mark_used(xconn, table, now); + + ngtcp2_path path; + path.remote.addr = (struct sockaddr *)dest; + path.remote.addrlen = addr_len((const struct sockaddr_in6 *)dest); + path.local.addr = (struct sockaddr *)via; + path.local.addrlen = addr_len((const struct sockaddr_in6 *)via); + + int ret = conn_new(&xconn->conn, &path, &dcid, &scid, &dcid, NGTCP2_PROTO_VER_V1, now, + table->udp_payload_limit, 5000000000L, xconn, false, false); + if (ret == KNOT_EOK) { + ret = tls_init_conn_session(xconn, false); + } + if (ret == KNOT_EOK) { + char *hostname = sockaddr_hostname(); + if (hostname == NULL) { + ret = KNOT_ENOMEM; + } else { + ret = gnutls_server_name_set(xconn->tls_session, GNUTLS_NAME_DNS, + hostname, strlen(hostname)); + free(hostname); + } + } + if (ret != KNOT_EOK) { + knot_xquic_table_rem(xconn, table); + return ret; + } + + *out_conn = xconn; + return KNOT_EOK; +} + +_public_ +int knot_xquic_handle(knot_xquic_table_t *table, knot_xdp_msg_t *msg, uint64_t idle_timeout, knot_xquic_conn_t **out_conn) +{ + *out_conn = NULL; + + ngtcp2_version_cid decoded_cids = { 0 }; + ngtcp2_cid scid = { 0 }, dcid = { 0 }, odcid = { 0 }; + uint64_t now = get_timestamp(); + int ret = ngtcp2_pkt_decode_version_cid(&decoded_cids, + msg->payload.iov_base, + msg->payload.iov_len, + SERVER_DEFAULT_SCIDLEN); + if (ret == NGTCP2_ERR_VERSION_NEGOTIATION) { + return -XQUIC_SEND_VERSION_NEGOTIATION; + } else if (ret != NGTCP2_NO_ERROR) { + return ret; + } + ngtcp2_cid_init(&dcid, decoded_cids.dcid, decoded_cids.dcidlen); + ngtcp2_cid_init(&scid, decoded_cids.scid, decoded_cids.scidlen); + + knot_xquic_conn_t *xconn = xquic_table_lookup(&dcid, table); + + if (decoded_cids.version == 0 /* short header */ && xconn == NULL) { + return KNOT_EOK; // NOOP + } + + ngtcp2_path path; + path.remote.addr = (struct sockaddr *)&msg->ip_from; + path.remote.addrlen = addr_len(&msg->ip_from); + path.local.addr = (struct sockaddr *)&msg->ip_to; + path.local.addrlen = addr_len(&msg->ip_to); + + if (xconn == NULL) { + // new conn + + ngtcp2_pkt_hd header = { 0 }; + ret = ngtcp2_accept(&header, msg->payload.iov_base, msg->payload.iov_len); + if (ret == NGTCP2_ERR_RETRY) { + return -XQUIC_SEND_RETRY; + } else if (ret != NGTCP2_NO_ERROR) { // discard packet + return KNOT_EOK; + } + + assert(header.type == NGTCP2_PKT_INITIAL); + if (header.tokenlen == 0 && xquic_require_retry(table)) { + return -XQUIC_SEND_RETRY; + } + + if (header.tokenlen > 0) { + ret = ngtcp2_crypto_verify_retry_token( + &odcid, header.token, header.tokenlen, + (const uint8_t *)table->hash_secret, sizeof(table->hash_secret), header.version, + (const struct sockaddr *)&msg->ip_from, addr_len(&msg->ip_from), + &dcid, idle_timeout, now // NOTE setting retry token validity to idle_timeout for simplicity + ); + if (ret != 0) { + return KNOT_EOK; + } + } else { + memcpy(&odcid, &dcid, sizeof(odcid)); + } + + // server chooses his CID to his liking + if (!init_unique_cid(&dcid, 0, table)) { + return KNOT_ERROR; + } + + xconn = xquic_table_add(NULL, &dcid, table); + if (xconn == NULL) { + return ENOMEM; + } + xquic_conn_mark_used(xconn, table, now); + + ret = conn_new(&xconn->conn, &path, &dcid, &scid, &odcid, decoded_cids.version, now, + table->udp_payload_limit, idle_timeout, xconn, true, header.tokenlen > 0); + if (ret >= 0) { + ret = tls_init_conn_session(xconn, true); + } + if (ret < 0) { + knot_xquic_table_rem(xconn, table); + return ret; + } + } + + ngtcp2_pkt_info pi = { .ecn = NGTCP2_ECN_NOT_ECT, }; + + ret = ngtcp2_conn_read_pkt(xconn->conn, &path, &pi, msg->payload.iov_base, msg->payload.iov_len, now); + + *out_conn = xconn; + if (ret == NGTCP2_ERR_DRAINING) { // received CONNECTION_CLOSE from the counterpart + knot_xquic_table_rem(xconn, table); + return KNOT_EOK; + } else if(ngtcp2_err_is_fatal(ret)) { // connection doomed + knot_xquic_table_rem(xconn, table); + return KNOT_ECONN; + } else if (ret != NGTCP2_NO_ERROR) { // non-fatal error, discard packet + return KNOT_EOK; + } + + xquic_conn_mark_used(xconn, table, now); + + return KNOT_EOK; +} + +static bool stream_exists(knot_xquic_conn_t *xconn, int64_t stream_id) +{ + // TRICK, we never use stream_user_data + return (ngtcp2_conn_set_stream_user_data(xconn->conn, stream_id, NULL) == NGTCP2_NO_ERROR); +} + +static int send_stream(knot_xquic_table_t *quic_table, knot_xdp_socket_t *sock, + knot_xdp_msg_t *in_msg, knot_xquic_conn_t *relay, int64_t stream_id, + uint8_t *data, size_t len, bool fin, ngtcp2_ssize *sent) +{ + (void)quic_table; + assert(stream_id >= 0 || (data == NULL && len == 0)); + + while (stream_id >= 0 && !stream_exists(relay, stream_id)) { + int64_t opened = 0; + int ret = ngtcp2_conn_open_bidi_stream(relay->conn, &opened, NULL); + if (ret != KNOT_EOK) { + return ret; + } + assert((bool)(opened == stream_id) == stream_exists(relay, stream_id)); + } + + uint32_t xdp_sent = 0; + knot_xdp_msg_t out_msg = { 0 }; + int ret = knot_xdp_reply_alloc(sock, in_msg, &out_msg); + if (ret != KNOT_EOK) { + return ret; + } + + uint32_t fl = ((stream_id >= 0 && fin) ? NGTCP2_WRITE_STREAM_FLAG_FIN : NGTCP2_WRITE_STREAM_FLAG_NONE); + ngtcp2_vec vec = { .base = data, .len = len }; + + ret = ngtcp2_conn_writev_stream(relay->conn, NULL, NULL, out_msg.payload.iov_base, out_msg.payload.iov_len, + sent, fl, stream_id, &vec, (stream_id >= 0 ? 1 : 0), get_timestamp()); + if (ret <= 0) { + knot_xdp_send_free(sock, &out_msg, 1); + return ret; + } + if (*sent < 0) { + *sent = 0; + } + + out_msg.payload.iov_len = ret; + ret = knot_xdp_send(sock, &out_msg, 1, &xdp_sent); + if (ret == KNOT_EOK) { + assert(xdp_sent == 1); + return 1; + } + return ret; +} + +static int send_special(knot_xquic_table_t *quic_table, knot_xdp_socket_t *sock, + knot_xdp_msg_t *in_msg, int handle_ret) +{ + knot_xdp_msg_t out_msg; + int ret = knot_xdp_reply_alloc(sock, in_msg, &out_msg); + if (ret != KNOT_EOK) { + return ret; + } + + uint64_t now = get_timestamp(); + ngtcp2_version_cid decoded_cids = { 0 }; + ngtcp2_cid scid = { 0 }, dcid = { 0 }; + + int dvc_ret = ngtcp2_pkt_decode_version_cid(&decoded_cids, + in_msg->payload.iov_base, + in_msg->payload.iov_len, + SERVER_DEFAULT_SCIDLEN); + + uint8_t rnd = 0; + dnssec_random_buffer(&rnd, sizeof(rnd)); + uint32_t supported_quic[1] = { NGTCP2_PROTO_VER_V1 }; + ngtcp2_cid new_dcid; + uint8_t retry_token[NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN]; + uint8_t stateless_reset_token[NGTCP2_STATELESS_RESET_TOKENLEN]; + uint8_t sreset_rand[NGTCP2_MIN_STATELESS_RESET_RANDLEN]; + dnssec_random_buffer(sreset_rand, sizeof(sreset_rand)); + + switch (handle_ret) { + case -XQUIC_SEND_VERSION_NEGOTIATION: + if (dvc_ret != NGTCP2_ERR_VERSION_NEGOTIATION) { + return KNOT_ERROR; + } + ret = ngtcp2_pkt_write_version_negotiation( + out_msg.payload.iov_base, out_msg.payload.iov_len, + rnd, decoded_cids.scid, decoded_cids.scidlen, decoded_cids.dcid, + decoded_cids.dcidlen, supported_quic, + sizeof(supported_quic) / sizeof(*supported_quic) + ); + break; + case -XQUIC_SEND_RETRY: + ngtcp2_cid_init(&dcid, decoded_cids.dcid, decoded_cids.dcidlen); + ngtcp2_cid_init(&scid, decoded_cids.scid, decoded_cids.scidlen); + + init_random_cid(&new_dcid, 0); + + ret = ngtcp2_crypto_generate_retry_token( + retry_token, (const uint8_t *)quic_table->hash_secret, sizeof(quic_table->hash_secret), decoded_cids.version, + (const struct sockaddr *)&in_msg->ip_from, sockaddr_len((const struct sockaddr_storage *)&in_msg->ip_from), + &new_dcid, &dcid, now + ); + + if (ret >= 0) { + ret = ngtcp2_crypto_write_retry( + out_msg.payload.iov_base, out_msg.payload.iov_len, + decoded_cids.version, &scid, &new_dcid, &dcid, retry_token, ret + ); + } + break; + case -XQUIC_SEND_STATELESS_RESET: + ret = ngtcp2_pkt_write_stateless_reset( + out_msg.payload.iov_base, out_msg.payload.iov_len, + stateless_reset_token, sreset_rand, sizeof(sreset_rand) + ); + break; + default: + ret = KNOT_EINVAL; + break; + } + + if (ret < 0) { + knot_xdp_send_free(sock, &out_msg, 1); + } else { + uint32_t sent; + out_msg.payload.iov_len = ret; + ret = knot_xdp_send(sock, &out_msg, 1, &sent); + } + return ret; +} + +_public_ +int knot_xquic_send(knot_xquic_table_t *quic_table, knot_xquic_conn_t *relay, + knot_xdp_socket_t *sock, knot_xdp_msg_t *in_msg, + int handle_ret, unsigned max_msgs, bool ignore_lastbyte) +{ + if (handle_ret < 0) { + return handle_ret; + } else if (handle_ret > 0) { + return send_special(quic_table, sock, in_msg, handle_ret); + } else if (relay == NULL) { + return KNOT_EINVAL; + } else if (relay->conn == NULL) { + return KNOT_EOK; + } + + unsigned sent_msgs = 0, stream_msgs = 0; + int ret = 1; + for (int64_t si = 0; si < relay->streams_count && sent_msgs < max_msgs; /* NO INCREMENT */) { + int64_t stream_id = 4 * (relay->streams_first + si); + + ngtcp2_ssize sent = 0; + size_t uf = relay->streams[si].unsent_offset; + knot_xquic_obuf_t *uo = relay->streams[si].unsent_obuf; + if (uo == NULL) { + si++; + continue; + } + + bool fin = (((node_t *)uo->node.next)->next == NULL) && !ignore_lastbyte; + ret = send_stream(quic_table, sock, in_msg, relay, stream_id, + uo->buf + uf, uo->len - uf - (ignore_lastbyte ? 1 : 0), + fin, &sent); + if (ret < 0) { + return ret; + } + + sent_msgs++; + stream_msgs++; + if (sent > 0 && ignore_lastbyte) { + sent++; + } + if (sent > 0) { + knot_xquic_stream_mark_sent(relay, stream_id, sent); + } + + if (stream_msgs >= max_msgs / relay->streams_count) { + stream_msgs = 0; + si++; // if this stream is sending too much, give chance to other streams + } + } + + while (ret == 1) { + ngtcp2_ssize unused = 0; + ret = send_stream(quic_table, sock, in_msg, relay, -1, NULL, 0, false, &unused); + } + + return ret; +} diff --git a/src/libknot/xdp/quic.h b/src/libknot/xdp/quic.h new file mode 100644 index 0000000..943a2f3 --- /dev/null +++ b/src/libknot/xdp/quic.h @@ -0,0 +1,134 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief General QUIC functionality. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include "libknot/xdp/quic_conn.h" +#include "libknot/xdp/xdp.h" + +struct knot_quic_creds; +struct knot_quic_session; + +/*! + * \brief Gets data needed for session resumption. + * + * \param conn QUIC connection. + * + * \return QUIC session context. + */ +struct knot_quic_session *knot_xquic_session_save(knot_xquic_conn_t *conn); + +/*! + * \brief Loads data needed for session resumption. + * + * \param conn QUIC connection. + * \param session QUIC session context. + * + * \return KNOT_E* + */ +int knot_xquic_session_load(knot_xquic_conn_t *conn, struct knot_quic_session *session); + +/*! + * \brief Init server TLS certificate for DoQ. + * + * \param server Initializing for server-side (client otherwise). + * \param tls_cert X509 certificate PEM file path/name. + * \param tls_key Key PEM file path/name. + * + * \return Initialized creds. + */ +struct knot_quic_creds *knot_xquic_init_creds(bool server, const char *tls_cert, + const char *tls_key); + +/*! + * \brief Init server TLS certificate for DoQ. + */ +void knot_xquic_free_creds(struct knot_quic_creds *creds); + +/*! + * \brief Returns timeout value for the connection. + */ +uint64_t xquic_conn_get_timeout(knot_xquic_conn_t *conn); + +/*! + * \brief Check if connection timed out due to inactivity. + * + * \param conn QUIC connection. + * \param now In/out: current monotonic time. Use zero first and reuse for + * next calls for optimization. + * + * \return True if the connection timed out idle. + */ +bool xquic_conn_timeout(knot_xquic_conn_t *conn, uint64_t *now); + +/*! + * \brief Returns measured connection RTT in usecs. + */ +uint32_t knot_xquic_conn_rtt(knot_xquic_conn_t *conn); + +/*! + * \brief Create new outgoing QUIC connection. + * + * \param table QUIC connections table to be added to. + * \param dest Destination IP address. + * \param via Source IP address. + * \param out_conn Out: new connection. + * + * \return KNOT_E* + */ +int knot_xquic_client(knot_xquic_table_t *table, struct sockaddr_in6 *dest, + struct sockaddr_in6 *via, knot_xquic_conn_t **out_conn); + +/*! + * \brief Handle incoming QUIC packet. + * + * \param table QUIC connectoins table- + * \param msg Incoming XDP packet. + * \param idle_timeout Configured idle timeout for connections (in nanoseconds). + * \param out_conn Out: QUIC connection that this packet belongs to. + * + * \return KNOT_E* + */ +int knot_xquic_handle(knot_xquic_table_t *table, knot_xdp_msg_t *msg, + uint64_t idle_timeout, knot_xquic_conn_t **out_conn); + +/*! + * \brief Send outgoing QUIC packet(s) for a connection. + * + * \param quic_table QUIC connection table. + * \param relay QUIC connection. + * \param sock XDP socket. + * \param in_msg Previous incomming packet for this connection. + * \param handle_ret Error returned from knot_xquic_handle() for incoming packet. + * \param max_msgs Maxmimum packets to be sent. + * \param ignore_lastbyte Cut off last byte of QUIC paylod. + * + * \return KNOT_E* + */ +int knot_xquic_send(knot_xquic_table_t *quic_table, knot_xquic_conn_t *relay, + knot_xdp_socket_t *sock, knot_xdp_msg_t *in_msg, + int handle_ret, unsigned max_msgs, bool ignore_lastbyte); + +/*! @} */ diff --git a/src/libknot/xdp/quic_conn.c b/src/libknot/xdp/quic_conn.c new file mode 100644 index 0000000..40a5d5b --- /dev/null +++ b/src/libknot/xdp/quic_conn.c @@ -0,0 +1,506 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <gnutls/gnutls.h> +#include <ngtcp2/ngtcp2.h> +#include <string.h> + +#include "libknot/xdp/quic_conn.h" + +#include "contrib/macros.h" +#include "contrib/openbsd/siphash.h" +#include "contrib/ucw/lists.h" +#include "libdnssec/random.h" +#include "libknot/attribute.h" +#include "libknot/error.h" +#include "libknot/xdp/quic.h" +#include "libknot/xdp/tcp_iobuf.h" +#include "libknot/wire.h" + +#define STREAM_INCR 4 // DoQ only uses client-initiated bi-directional streams, so stream IDs increment by four +#define BUCKETS_PER_CONNS 8 // Each connecion has several dCIDs, and each CID takes one hash table bucket. + +_public_ +knot_xquic_table_t *knot_xquic_table_new(size_t max_conns, size_t max_ibufs, size_t max_obufs, + size_t udp_payload, struct knot_quic_creds *creds) +{ + size_t table_size = max_conns * BUCKETS_PER_CONNS; + + knot_xquic_table_t *res = calloc(1, sizeof(*res) + table_size * sizeof(res->conns[0])); + if (res == NULL) { + return NULL; + } + + res->size = table_size; + res->max_conns = max_conns; + res->ibufs_max = max_ibufs; + res->obufs_max = max_obufs; + res->udp_payload_limit = udp_payload; + init_list((list_t *)&res->timeout); + + res->creds = creds; + + res->hash_secret[0] = dnssec_random_uint64_t(); + res->hash_secret[1] = dnssec_random_uint64_t(); + res->hash_secret[2] = dnssec_random_uint64_t(); + res->hash_secret[3] = dnssec_random_uint64_t(); + + return res; +} + +_public_ +void knot_xquic_table_free(knot_xquic_table_t *table) +{ + if (table != NULL) { + knot_xquic_conn_t *c, *next; + list_t *tto = (list_t *)&table->timeout; + WALK_LIST_DELSAFE(c, next, *tto) { + knot_xquic_table_rem(c, table); + knot_xquic_cleanup(&c, 1); + } + assert(table->usage == 0); + assert(table->pointers == 0); + assert(table->obufs_size == 0); + + free(table); + } +} + +_public_ +int knot_xquic_table_sweep(knot_xquic_table_t *table, struct knot_sweep_stats *stats) +{ + uint64_t now = 0; + knot_xquic_conn_t *c, *next; + list_t *tto = (list_t *)&table->timeout; + WALK_LIST_DELSAFE(c, next, *tto) { + if (xquic_conn_timeout(c, &now)) { + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_TIMEOUT); + knot_xquic_table_rem(c, table); + } else if (table->usage > table->max_conns) { + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_CONN); + knot_xquic_table_rem(c, table); + // NOTE here it would be correct to send Immediate close + // with DoQ errcode DOQ_EXCESSIVE_LOAD + // nowever, we don't do this for the sake of simplicty + // it would be possible to send by using ngtcp2_conn_get_path()... + // (also applies to below case) + } else if (table->obufs_size > table->obufs_max) { + if (c->obufs_size > 0) { + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_OBUF); + knot_xquic_table_rem(c, table); + } + } else if (table->ibufs_size > table->ibufs_max) { + if (c->ibufs_size > 0) { + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_IBUF); + knot_xquic_table_rem(c, table); + } + } else { + break; + } + knot_xquic_cleanup(&c, 1); + } + return KNOT_EOK; +} + +static uint64_t cid2hash(const ngtcp2_cid *cid, knot_xquic_table_t *table) +{ + SIPHASH_CTX ctx; + SipHash24_Init(&ctx, (const SIPHASH_KEY *)(table->hash_secret)); + SipHash24_Update(&ctx, cid->data, MIN(cid->datalen, 8)); + uint64_t ret = SipHash24_End(&ctx); + return ret; +} + +knot_xquic_cid_t **xquic_table_insert(knot_xquic_conn_t *xconn, const ngtcp2_cid *cid, + knot_xquic_table_t *table) +{ + uint64_t hash = cid2hash(cid, table); + + knot_xquic_cid_t *cidobj = malloc(sizeof(*cidobj)); + if (cidobj == NULL) { + return NULL; + } + _Static_assert(sizeof(*cid) <= sizeof(cidobj->cid_placeholder), "insufficient placeholder for CID struct"); + memcpy(cidobj->cid_placeholder, cid, sizeof(*cid)); + cidobj->conn = xconn; + + knot_xquic_cid_t **addto = table->conns + (hash % table->size); + + cidobj->next = *addto; + *addto = cidobj; + table->pointers++; + + return addto; +} + +knot_xquic_conn_t *xquic_table_add(ngtcp2_conn *conn, const ngtcp2_cid *cid, + knot_xquic_table_t *table) +{ + knot_xquic_conn_t *xconn = calloc(1, sizeof(*xconn)); + if (xconn == NULL) { + return NULL; + } + + xconn->conn = conn; + xconn->xquic_table = table; + xconn->stream_inprocess = -1; + + knot_xquic_cid_t **addto = xquic_table_insert(xconn, cid, table); + if (addto == NULL) { + free(xconn); + return NULL; + } + table->usage++; + + return xconn; +} + +knot_xquic_cid_t **xquic_table_lookup2(const ngtcp2_cid *cid, knot_xquic_table_t *table) +{ + uint64_t hash = cid2hash(cid, table); + + knot_xquic_cid_t **res = table->conns + (hash % table->size); + while (*res != NULL && !ngtcp2_cid_eq(cid, (const ngtcp2_cid *)(*res)->cid_placeholder)) { + res = &(*res)->next; + } + return res; +} + +knot_xquic_conn_t *xquic_table_lookup(const ngtcp2_cid *cid, knot_xquic_table_t *table) +{ + knot_xquic_cid_t **pcid = xquic_table_lookup2(cid, table); + assert(pcid != NULL); + return *pcid == NULL ? NULL : (*pcid)->conn; +} + +void xquic_conn_mark_used(knot_xquic_conn_t *conn, knot_xquic_table_t *table, + uint64_t now) +{ + node_t *n = (node_t *)&conn->timeout; + list_t *l = (list_t *)&table->timeout; + if (n->next != NULL) { + rem_node(n); + } + add_tail(l, n); + conn->last_ts = now; +} + +void xquic_table_rem2(knot_xquic_cid_t **pcid, knot_xquic_table_t *table) +{ + knot_xquic_cid_t *cid = *pcid; + *pcid = cid->next; + free(cid); + table->pointers--; +} + +void xquic_stream_free(knot_xquic_conn_t *xconn, int64_t stream_id) +{ + knot_xquic_stream_ack_data(xconn, stream_id, SIZE_MAX, false); +} + +_public_ +void knot_xquic_table_rem(knot_xquic_conn_t *conn, knot_xquic_table_t *table) +{ + if (conn->streams_count == -1) { // kxdpgun special + conn->streams_count = 1; + } + for (ssize_t i = conn->streams_count - 1; i >= 0; i--) { + xquic_stream_free(conn, (i + conn->streams_first) * 4); + } + assert(conn->streams_count <= 0); + assert(conn->obufs_size == 0); + + size_t num_scid = ngtcp2_conn_get_num_scid(conn->conn); + ngtcp2_cid *scids = calloc(num_scid, sizeof(*scids)); + ngtcp2_conn_get_scid(conn->conn, scids); + + for (size_t i = 0; i < num_scid; i++) { + knot_xquic_cid_t **pcid = xquic_table_lookup2(&scids[i], table); + assert(pcid != NULL); + if (*pcid == NULL) { + continue; + } + assert((*pcid)->conn == conn); + xquic_table_rem2(pcid, table); + } + + rem_node((node_t *)&conn->timeout); + + free(scids); + + gnutls_deinit(conn->tls_session); + ngtcp2_conn_del(conn->conn); + conn->conn = NULL; + + table->usage--; +} + +_public_ +knot_xquic_stream_t *knot_xquic_conn_get_stream(knot_xquic_conn_t *xconn, + int64_t stream_id, bool create) +{ + if (stream_id % 4 != 0) { + return NULL; + } + stream_id /= 4; + + if (xconn->streams_first > stream_id) { + return NULL; + } + if (xconn->streams_count > stream_id - xconn->streams_first) { + return &xconn->streams[stream_id - xconn->streams_first]; + } + + if (create) { + size_t new_streams_count; + knot_xquic_stream_t *new_streams; + + if (xconn->streams_count == 0) { + new_streams = malloc(sizeof(new_streams[0])); + if (new_streams == NULL) { + return NULL; + } + new_streams_count = 1; + xconn->streams_first = stream_id; + } else { + new_streams_count = stream_id + 1 - xconn->streams_first; + if (new_streams_count > MAX_STREAMS_PER_CONN) { + return NULL; + } + new_streams = realloc(xconn->streams, new_streams_count * sizeof(*new_streams)); + if (new_streams == NULL) { + return NULL; + } + } + + for (knot_xquic_stream_t *si = new_streams; + si < new_streams + xconn->streams_count; si++) { + if (si->obufs_size == 0) { + init_list((list_t *)&si->outbufs); + } else { + fix_list((list_t *)&si->outbufs); + } + } + + for (knot_xquic_stream_t *si = new_streams + xconn->streams_count; + si < new_streams + new_streams_count; si++) { + memset(si, 0, sizeof(*si)); + init_list((list_t *)&si->outbufs); + } + xconn->streams = new_streams; + xconn->streams_count = new_streams_count; + + return &xconn->streams[stream_id - xconn->streams_first]; + } + return NULL; +} + +static void stream_inprocess(knot_xquic_conn_t *xconn, knot_xquic_stream_t *stream) +{ + int16_t idx = stream - xconn->streams; + assert(idx >= 0); + assert(idx < xconn->streams_count); + if (xconn->stream_inprocess < 0 || xconn->stream_inprocess > idx) { + xconn->stream_inprocess = idx; + } +} + +static void stream_outprocess(knot_xquic_conn_t *xconn, knot_xquic_stream_t *stream) +{ + if (stream != &xconn->streams[xconn->stream_inprocess]) { + return; + } + + for (int16_t idx = xconn->stream_inprocess + 1; idx < xconn->streams_count; idx++) { + stream = &xconn->streams[idx]; + if (stream->inbuf_fin != NULL) { + xconn->stream_inprocess = stream - xconn->streams; + return; + } + } + xconn->stream_inprocess = -1; +} + +int knot_xquic_stream_recv_data(knot_xquic_conn_t *xconn, int64_t stream_id, + const uint8_t *data, size_t len, bool fin) +{ + if (len == 0) { + return KNOT_EINVAL; + } + + knot_xquic_stream_t *stream = knot_xquic_conn_get_stream(xconn, stream_id, true); + if (stream == NULL) { + return KNOT_ENOENT; + } + + struct iovec in = { (void *)data, len }, *outs; + size_t outs_count; + int ret = knot_tcp_inbuf_update(&stream->inbuf, in, &outs, &outs_count, + &xconn->ibufs_size); + if (ret != KNOT_EOK || (outs_count == 0 && !fin)) { + return ret; + } + if (outs_count != 1 || !fin) { + free(outs); + return KNOT_ESEMCHECK; + } + + stream->inbuf_fin = outs; + stream_inprocess(xconn, stream); + return KNOT_EOK; +} + +_public_ +knot_xquic_stream_t *knot_xquic_stream_get_process(knot_xquic_conn_t *xconn, + int64_t *stream_id) +{ + if (xconn->stream_inprocess < 0) { + return NULL; + } + + knot_xquic_stream_t *stream = &xconn->streams[xconn->stream_inprocess]; + *stream_id = (xconn->streams_first + xconn->stream_inprocess) * 4; + stream_outprocess(xconn, stream); + return stream; +} + +_public_ +uint8_t *knot_xquic_stream_add_data(knot_xquic_conn_t *xconn, int64_t stream_id, + uint8_t *data, size_t len) +{ + knot_xquic_stream_t *s = knot_xquic_conn_get_stream(xconn, stream_id, true); + if (s == NULL) { + return NULL; + } + + size_t prefix = sizeof(uint16_t); + + knot_xquic_obuf_t *obuf = malloc(sizeof(*obuf) + prefix + len); + if (obuf == NULL) { + return NULL; + } + + obuf->len = len + prefix; + knot_wire_write_u16(obuf->buf, len); + if (data != NULL) { + memcpy(obuf->buf + prefix, data, len); + } + + list_t *list = (list_t *)&s->outbufs; + if (EMPTY_LIST(*list)) { + s->unsent_obuf = obuf; + } + add_tail((list_t *)&s->outbufs, (node_t *)obuf); + s->obufs_size += obuf->len; + xconn->obufs_size += obuf->len; + xconn->xquic_table->obufs_size += obuf->len; + + return obuf->buf + prefix; +} + +void knot_xquic_stream_ack_data(knot_xquic_conn_t *xconn, int64_t stream_id, + size_t end_acked, bool keep_stream) +{ + knot_xquic_stream_t *s = knot_xquic_conn_get_stream(xconn, stream_id, false); + if (s == NULL) { + return; + } + + list_t *obs = (list_t *)&s->outbufs; + + knot_xquic_obuf_t *first; + while (!EMPTY_LIST(*obs) && end_acked >= (first = HEAD(*obs))->len + s->first_offset) { + rem_node((node_t *)first); + s->obufs_size -= first->len; + xconn->obufs_size -= first->len; + xconn->xquic_table->obufs_size -= first->len; + s->first_offset += first->len; + free(first); + if (s->unsent_obuf == first) { + s->unsent_obuf = EMPTY_LIST(*obs) ? NULL : HEAD(*obs); + s->unsent_offset = 0; + } + } + + if (EMPTY_LIST(*obs) && !keep_stream) { + stream_outprocess(xconn, s); + memset(s, 0, sizeof(*s)); + while (s = &xconn->streams[0], s->inbuf.iov_len == 0 && s->inbuf_fin == NULL && s->obufs_size == 0) { + assert(xconn->streams_count > 0); + xconn->streams_count--; + + if (xconn->streams_count == 0) { + free(xconn->streams); + xconn->streams = 0; + xconn->streams_first = 0; + break; + } else { + xconn->streams_first++; + xconn->stream_inprocess--; + memmove(s, s + 1, sizeof(*s) * xconn->streams_count); + // possible realloc to shrink allocated space, but probably useless + for (knot_xquic_stream_t *si = s; si < s + xconn->streams_count; si++) { + if (si->obufs_size == 0) { + init_list((list_t *)&si->outbufs); + } else { + fix_list((list_t *)&si->outbufs); + } + } + } + } + } +} + +void knot_xquic_stream_mark_sent(knot_xquic_conn_t *xconn, int64_t stream_id, + size_t amount_sent) +{ + knot_xquic_stream_t *s = knot_xquic_conn_get_stream(xconn, stream_id, false); + if (s == NULL) { + return; + } + + s->unsent_offset += amount_sent; + assert(s->unsent_offset <= s->unsent_obuf->len); + if (s->unsent_offset == s->unsent_obuf->len) { + s->unsent_offset = 0; + s->unsent_obuf = (knot_xquic_obuf_t *)s->unsent_obuf->node.next; + if (s->unsent_obuf->node.next == NULL) { // already behind the tail of list + s->unsent_obuf = NULL; + } + } +} + +_public_ +void knot_xquic_cleanup(knot_xquic_conn_t *conns[], size_t n_conns) +{ + for (size_t i = 0; i < n_conns; i++) { + if (conns[i] != NULL && conns[i]->conn == NULL) { + free(conns[i]); + for (size_t j = i + 1; j < n_conns; j++) { + if (conns[j] == conns[i]) { + conns[j] = NULL; + } + } + } + } +} + +bool xquic_require_retry(knot_xquic_table_t *table) +{ + (void)table; + return false; +} diff --git a/src/libknot/xdp/quic_conn.h b/src/libknot/xdp/quic_conn.h new file mode 100644 index 0000000..20ee176 --- /dev/null +++ b/src/libknot/xdp/quic_conn.h @@ -0,0 +1,314 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief QUIC connection management. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <linux/if_ether.h> +#include <stdbool.h> +#include <stdint.h> +#include <sys/uio.h> + +#define MAX_STREAMS_PER_CONN 10 + +struct ngtcp2_cid; // declaration taken from wherever in ngtcp2 +struct knot_quic_creds; +struct knot_sweep_stats; + +// those are equivalent to contrib/ucw/lists.h , just must not be included. +typedef struct knot_xquic_ucw_node { + struct knot_xquic_ucw_node *next, *prev; +} knot_xquic_ucw_node_t; +typedef struct knot_xquic_ucw_list { + knot_xquic_ucw_node_t head, tail; +} knot_xquic_ucw_list_t; + +typedef struct { + void *get_conn; + void *user_data; +} nc_conn_ref_placeholder_t; + +typedef struct { + knot_xquic_ucw_node_t node; + size_t len; + uint8_t buf[]; +} knot_xquic_obuf_t; + +typedef struct { + struct iovec inbuf; + struct iovec *inbuf_fin; + knot_xquic_ucw_list_t outbufs; + size_t obufs_size; + + knot_xquic_obuf_t *unsent_obuf; + size_t first_offset; + size_t unsent_offset; +} knot_xquic_stream_t; + +typedef struct knot_xquic_conn { + knot_xquic_ucw_node_t timeout; // MUST be first field of the struct + uint64_t last_ts; + + nc_conn_ref_placeholder_t conn_ref; // placeholder for internal struct ngtcp2_crypto_conn_ref + + struct ngtcp2_conn *conn; + + struct gnutls_session_int *tls_session; + + knot_xquic_stream_t *streams; + int16_t streams_count; // number of allocated streams structures + int16_t stream_inprocess; // index of first stream that has complete incomming data to be processed (aka inbuf_fin) + bool handshake_done; + bool session_taken; // TODO ... ? + int64_t streams_first; // stream_id/4 of first allocated stream + size_t ibufs_size; + size_t obufs_size; + + struct knot_xquic_table *xquic_table; + + struct knot_xquic_conn *next; +} knot_xquic_conn_t; + +typedef struct knot_xquic_cid { + uint8_t cid_placeholder[32]; + knot_xquic_conn_t *conn; + struct knot_xquic_cid *next; +} knot_xquic_cid_t; + +typedef struct knot_xquic_table { + size_t size; + size_t usage; + size_t pointers; + size_t max_conns; + size_t ibufs_max; + size_t obufs_max; + size_t ibufs_size; + size_t obufs_size; + size_t udp_payload_limit; // for simplicity not distinguishing IPv4/6 + void (*log_cb)(const char *); + uint64_t hash_secret[4]; + struct knot_quic_creds *creds; + knot_xquic_ucw_list_t timeout; + knot_xquic_cid_t *conns[]; +} knot_xquic_table_t; + +/*! + * \brief Allocate QUIC connections hash table. + * + * \param max_conns Maximum nuber of connections. + * \param max_ibufs Maximum size of buffers for fragmented incomming DNS msgs. + * \param max_obufs Maximum size of buffers for un-ACKed outgoing data. + * \param udp_pl Maximum UDP payload size (both IPv4 and 6). + * \param creds QUIC crypto context.. + * + * \return Allocated table, or NULL. + */ +knot_xquic_table_t *knot_xquic_table_new(size_t max_conns, size_t max_ibufs, size_t max_obufs, + size_t udp_payload, struct knot_quic_creds *creds); + +/*! + * \brief Free QUIC table including its contents. + * + * \param table Table to be freed. + */ +void knot_xquic_table_free(knot_xquic_table_t *table); + +/*! + * \brief Close timed out connections and some oldest ones if table full. + * + * \param table QUIC table to be cleaned up. + * \param stats Out: sweep statistics. + * + * \return KNOT_E* + */ +int knot_xquic_table_sweep(knot_xquic_table_t *table, struct knot_sweep_stats *stats); + +/*! + * \brief Add new connection/CID link to table. + * + * \param xconn QUIC connection linked. + * \param cid New CID to be added. + * \param table QUIC table to be modified. + * + * \return Pointer on the CID reference in table, or NULL. + */ +knot_xquic_cid_t **xquic_table_insert(knot_xquic_conn_t *xconn, + const struct ngtcp2_cid *cid, + knot_xquic_table_t *table); + +/*! + * \brief Add new connection to the table, allocating conn struct. + * + * \param conn Ngtcp2 conn struct. + * \param cid CID to be linked (usually oscid for server). + * \param table QUIC table to be modified. + * + * \return Allocated (and linked) Knot conn struct, or NULL. + */ +knot_xquic_conn_t *xquic_table_add(struct ngtcp2_conn *conn, + const struct ngtcp2_cid *cid, + knot_xquic_table_t *table); + +/*! + * \brief Lookup connection/CID link in table. + * + * \param cid CID to be searched for. + * \param table QUIC table. + * + * \return Pointer on the CID reference in table, or NULL. + */ +knot_xquic_cid_t **xquic_table_lookup2(const struct ngtcp2_cid *cid, + knot_xquic_table_t *table); + +/*! + * \brief Lookup QUIC connection in table. + * + * \param cid CID to be searched for. + * \param table QUIC table. + * + * \return Connection that the CID belongs to, or NULL. + */ +knot_xquic_conn_t *xquic_table_lookup(const struct ngtcp2_cid *cid, + knot_xquic_table_t *table); + +/*! + * \brief Put the connection on the end of timeout queue. + */ +void xquic_conn_mark_used(knot_xquic_conn_t *conn, knot_xquic_table_t *table, + uint64_t now); + +/*! + * \brief Remove connection/CID link from table. + * + * \param pcid CID to be removed. + * \param table QUIC table. + */ +void xquic_table_rem2(knot_xquic_cid_t **pcid, knot_xquic_table_t *table); + +/*! + * \brief Remove specified stream from QUIC connection, freeing all buffers. + * + * \param xconn QUIC connection to remove from. + * \param stream_id Stream QUIC ID. + */ +void xquic_stream_free(knot_xquic_conn_t *xconn, int64_t stream_id); + +/*! + * \brief Remove and deinitialize connection completely. + * + * \param conn Connection to be removed. + * \param table Table to remove from. + */ +void knot_xquic_table_rem(knot_xquic_conn_t *conn, knot_xquic_table_t *table); + +/*! + * \brief Fetch or initialize a QUIC stream. + * + * \param xconn QUIC connection. + * \param stream_id Stream QUIC ID. + * \param create Trigger stream creation if not exists. + * + * \return Stream or NULL. + */ +knot_xquic_stream_t *knot_xquic_conn_get_stream(knot_xquic_conn_t *xconn, + int64_t stream_id, bool create); + +/*! + * \brief Process incomming stream data to stream structure. + * + * \param xconn QUIC connection that has received data. + * \param stream_id Stream QUIC ID of the incomming data. + * \param data Incomming payload data. + * \param len Incomming payload data length. + * \param fin FIN flag set for incomming data. + * + * \return KNOT_E* + */ +int knot_xquic_stream_recv_data(knot_xquic_conn_t *xconn, int64_t stream_id, + const uint8_t *data, size_t len, bool fin); + +/*! + * \brief Get next stream which has pending incomming data to be processed. + * + * \param xconn QUIC connection. + * \param stream_id Out: strem QUIC ID of the returned stream. + * + * \return Stream with incomming data. + */ +knot_xquic_stream_t *knot_xquic_stream_get_process(knot_xquic_conn_t *xconn, + int64_t *stream_id); + +/*! + * \brief Add outgiong data to the stream for sending. + * + * \param xconn QUIC connection that shall send data. + * \param stream_id Stream ID for outgoing data. + * \param data Data payload. + * \param len Data payload length. + * + * \return NULL if error, or pinter at the data in outgiong buffer. + */ +uint8_t *knot_xquic_stream_add_data(knot_xquic_conn_t *xconn, int64_t stream_id, + uint8_t *data, size_t len); + +/*! + * \brief Mark outgiong data as acknowledged after ACK received. + * + * \param xconn QUIC connection that received ACK. + * \param stream_id Stream ID of ACKed data. + * \param end_acked Offset of ACKed data + ACKed length. + * \param keep_stream Don't free the stream even when ACKed all outgoing data. + */ +void knot_xquic_stream_ack_data(knot_xquic_conn_t *xconn, int64_t stream_id, + size_t end_acked, bool keep_stream); + +/*! + * \brief Mark outgoing data as sent. + * + * \param xconn QUIC connection that sent data. + * \param stream_id Stream ID of sent data. + * \param amount_sent Length of sent data. + */ +void knot_xquic_stream_mark_sent(knot_xquic_conn_t *xconn, int64_t stream_id, + size_t amount_sent); + +/*! + * \brief Free rest of resources of closed conns. + * + * \param conns Array with recently used conns (possibly NULLs). + * \param n_conns Size of the array. + */ +void knot_xquic_cleanup(knot_xquic_conn_t *conns[], size_t n_conns); + +/*! + * \brief Toggle sending Retry packet as a reaction to Initial packet of new connection. + * + * \param table Connection table. + * + * \return True if instead of continuing handshake, Retry packet shall be sent + * to verify counterpart's address. + */ +bool xquic_require_retry(knot_xquic_table_t *table); + +/*! @} */ diff --git a/src/libknot/xdp/tcp.c b/src/libknot/xdp/tcp.c new file mode 100644 index 0000000..752e9ef --- /dev/null +++ b/src/libknot/xdp/tcp.c @@ -0,0 +1,729 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "libknot/xdp/tcp.h" +#include "libknot/xdp/tcp_iobuf.h" +#include "libknot/attribute.h" +#include "libknot/error.h" +#include "libdnssec/random.h" +#include "contrib/macros.h" +#include "contrib/openbsd/siphash.h" +#include "contrib/ucw/lists.h" + +static uint32_t get_timestamp(void) +{ + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t res = (uint64_t)t.tv_sec * 1000000; + res += (uint64_t)t.tv_nsec / 1000; + return res & 0xffffffff; // overflow does not matter since we are working with differences +} + +static size_t sockaddr_data_len(const struct sockaddr_in6 *rem, const struct sockaddr_in6 *loc) +{ + assert(rem->sin6_family == loc->sin6_family); + if (rem->sin6_family == AF_INET) { + return offsetof(struct sockaddr_in, sin_zero); + } else { + assert(rem->sin6_family == AF_INET6); + return offsetof(struct sockaddr_in6, sin6_scope_id); + } +} + +static uint64_t hash_four_tuple(const struct sockaddr_in6 *rem, const struct sockaddr_in6 *loc, + knot_tcp_table_t *table) +{ + size_t socka_data_len = sockaddr_data_len(rem, loc); + SIPHASH_CTX ctx; + SipHash24_Init(&ctx, (const SIPHASH_KEY *)(table->hash_secret)); + SipHash24_Update(&ctx, rem, socka_data_len); + SipHash24_Update(&ctx, loc, socka_data_len); + return SipHash24_End(&ctx); +} + +static list_t *tcp_table_timeout(knot_tcp_table_t *table) +{ + return (list_t *)&table->conns[table->size]; +} + +static node_t *tcp_conn_node(knot_tcp_conn_t *conn) +{ + return (node_t *)&conn->list_node_placeholder; +} + +static void next_node_ptr(knot_tcp_conn_t **ptr) +{ + if (*ptr != NULL) { + *ptr = (*ptr)->list_node_placeholder.list_node_next; + if ((*ptr)->list_node_placeholder.list_node_next == NULL) { // detected tail of list + *ptr = NULL; + } + } +} + +static void next_ptr_ibuf(knot_tcp_conn_t **ptr) +{ + do { + next_node_ptr(ptr); + } while (*ptr != NULL && (*ptr)->inbuf.iov_len == 0); +} + +static void next_ptr_obuf(knot_tcp_conn_t **ptr) +{ + do { + next_node_ptr(ptr); + } while (*ptr != NULL && knot_tcp_outbufs_usage((*ptr)->outbufs) == 0); +} + +_public_ +knot_tcp_table_t *knot_tcp_table_new(size_t size, knot_tcp_table_t *secret_share) +{ + knot_tcp_table_t *table = calloc(1, sizeof(*table) + sizeof(list_t) + + size * sizeof(table->conns[0])); + if (table == NULL) { + return table; + } + + table->size = size; + init_list(tcp_table_timeout(table)); + + assert(sizeof(table->hash_secret) == sizeof(SIPHASH_KEY)); + if (secret_share == NULL) { + table->hash_secret[0] = dnssec_random_uint64_t(); + table->hash_secret[1] = dnssec_random_uint64_t(); + } else { + table->hash_secret[0] = secret_share->hash_secret[0]; + table->hash_secret[1] = secret_share->hash_secret[1]; + } + + return table; +} + +static void del_conn(knot_tcp_conn_t *conn) +{ + if (conn != NULL) { + free(conn->inbuf.iov_base); + while (conn->outbufs != NULL) { + struct knot_tcp_outbuf *next = conn->outbufs->next; + free(conn->outbufs); + conn->outbufs = next; + } + free(conn); + } +} + +_public_ +void knot_tcp_table_free(knot_tcp_table_t *table) +{ + if (table != NULL) { + knot_tcp_conn_t *conn, *next; + WALK_LIST_DELSAFE(conn, next, *tcp_table_timeout(table)) { + del_conn(conn); + } + free(table); + } +} + +static knot_tcp_conn_t **tcp_table_lookup(const struct sockaddr_in6 *rem, + const struct sockaddr_in6 *loc, + uint64_t *hash, knot_tcp_table_t *table) +{ + if (*hash == 0) { + *hash = hash_four_tuple(rem, loc, table); + } + size_t sdl = sockaddr_data_len(rem, loc); + knot_tcp_conn_t **res = table->conns + (*hash % table->size); + while (*res != NULL) { + if (memcmp(&(*res)->ip_rem, rem, sdl) == 0 && + memcmp(&(*res)->ip_loc, loc, sdl) == 0) { + break; + } + res = &(*res)->next; + } + return res; +} + +static knot_tcp_conn_t **tcp_table_re_lookup(knot_tcp_conn_t *conn, + knot_tcp_table_t *table) +{ + uint64_t unused_hash = 0; + knot_tcp_conn_t **res = tcp_table_lookup(&conn->ip_rem, &conn->ip_loc, + &unused_hash, table); + assert(*res == conn); + return res; +} + +static void rem_align_pointers(knot_tcp_conn_t *to_rem, knot_tcp_table_t *table) +{ + if (to_rem == table->next_close) { + next_node_ptr(&table->next_close); + } + if (to_rem == table->next_ibuf) { + next_ptr_ibuf(&table->next_ibuf); + } + if (to_rem == table->next_obuf) { + next_ptr_obuf(&table->next_obuf); + } + if (to_rem == table->next_resend) { + next_ptr_obuf(&table->next_resend); + } +} + +static void tcp_table_remove_conn(knot_tcp_conn_t **todel) +{ + rem_node(tcp_conn_node(*todel)); // remove from timeout double-linked list + *todel = (*todel)->next; // remove from conn-table linked list +} + +static void tcp_table_remove(knot_tcp_conn_t **todel, knot_tcp_table_t *table) +{ + assert(table->usage > 0); + rem_align_pointers(*todel, table); + table->inbufs_total -= (*todel)->inbuf.iov_len; + table->outbufs_total -= knot_tcp_outbufs_usage((*todel)->outbufs); + tcp_table_remove_conn(todel); + table->usage--; +} + +static void conn_init_from_msg(knot_tcp_conn_t *conn, knot_xdp_msg_t *msg) +{ + memcpy(&conn->ip_rem, &msg->ip_from, sizeof(conn->ip_rem)); + memcpy(&conn->ip_loc, &msg->ip_to, sizeof(conn->ip_loc)); + + memcpy(&conn->last_eth_rem, &msg->eth_from, sizeof(conn->last_eth_rem)); + memcpy(&conn->last_eth_loc, &msg->eth_to, sizeof(conn->last_eth_loc)); + + conn->seqno = msg->seqno; + conn->ackno = msg->ackno; + conn->acked = msg->ackno; + + conn->last_active = get_timestamp(); + conn->state = XDP_TCP_NORMAL; + conn->establish_rtt = 0; + + memset(&conn->inbuf, 0, sizeof(conn->inbuf)); + memset(&conn->outbufs, 0, sizeof(conn->outbufs)); +} + +static void tcp_table_insert(knot_tcp_conn_t *conn, uint64_t hash, + knot_tcp_table_t *table) +{ + knot_tcp_conn_t **addto = table->conns + (hash % table->size); + add_tail(tcp_table_timeout(table), tcp_conn_node(conn)); + if (table->next_close == NULL) { + table->next_close = conn; + } + conn->next = *addto; + *addto = conn; + table->usage++; +} + +// WARNING you shall ensure that it's not in the table already! +static int tcp_table_add(knot_xdp_msg_t *msg, uint64_t hash, knot_tcp_table_t *table, + knot_tcp_conn_t **res) +{ + knot_tcp_conn_t *c = malloc(sizeof(*c)); + if (c == NULL) { + return KNOT_ENOMEM; + } + conn_init_from_msg(c, msg); + tcp_table_insert(c, hash, table); + *res = c; + return KNOT_EOK; +} + +static bool check_seq_ack(const knot_xdp_msg_t *msg, const knot_tcp_conn_t *conn) +{ + if (conn == NULL || conn->seqno != msg->seqno) { + return false; + } + + if (conn->acked <= conn->ackno) { // ackno does not wrap around uint32 + return (msg->ackno >= conn->acked && msg->ackno <= conn->ackno); + } else { // this is more tricky + return (msg->ackno >= conn->acked || msg->ackno <= conn->ackno); + } +} + +static void conn_update(knot_tcp_conn_t *conn, const knot_xdp_msg_t *msg) +{ + conn->seqno = knot_tcp_next_seqno(msg); + memcpy(conn->last_eth_rem, msg->eth_from, sizeof(conn->last_eth_rem)); + memcpy(conn->last_eth_loc, msg->eth_to, sizeof(conn->last_eth_loc)); + conn->window_size = (uint32_t)msg->win * (1LU << conn->window_scale); + + uint32_t now = get_timestamp(); + if (conn->establish_rtt == 0 && conn->last_active != 0) { + conn->establish_rtt = now - conn->last_active; + } + conn->last_active = now; +} + +_public_ +int knot_tcp_recv(knot_tcp_relay_t *relays, knot_xdp_msg_t msgs[], uint32_t msg_count, + knot_tcp_table_t *tcp_table, knot_tcp_table_t *syn_table, + knot_tcp_ignore_t ignore) +{ + if (msg_count == 0) { + return KNOT_EOK; + } + if (relays == NULL || msgs == NULL || tcp_table == NULL) { + return KNOT_EINVAL; + } + memset(relays, 0, msg_count * sizeof(*relays)); + + knot_tcp_relay_t *relay = relays; + int ret = KNOT_EOK; + + for (knot_xdp_msg_t *msg = msgs; msg != msgs + msg_count && ret == KNOT_EOK; msg++) { + if (!(msg->flags & KNOT_XDP_MSG_TCP)) { + continue; + } + + uint64_t conn_hash = 0; + knot_tcp_conn_t **pconn = tcp_table_lookup(&msg->ip_from, &msg->ip_to, + &conn_hash, tcp_table); + knot_tcp_conn_t *conn = *pconn; + bool seq_ack_match = check_seq_ack(msg, conn); + if (seq_ack_match) { + assert(conn->mss != 0); + conn_update(conn, msg); + + rem_align_pointers(conn, tcp_table); + rem_node(tcp_conn_node(conn)); + add_tail(tcp_table_timeout(tcp_table), tcp_conn_node(conn)); + + if (msg->flags & KNOT_XDP_MSG_ACK) { + conn->acked = msg->ackno; + knot_tcp_outbufs_ack(&conn->outbufs, msg->ackno, &tcp_table->outbufs_total); + } + } + + relay->msg = msg; + relay->conn = conn; + + // process incoming data + if (seq_ack_match && (msg->flags & KNOT_XDP_MSG_ACK) && msg->payload.iov_len > 0) { + if (!(ignore & XDP_TCP_IGNORE_DATA_ACK)) { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + ret = knot_tcp_inbuf_update(&conn->inbuf, msg->payload, &relay->inbufs, + &relay->inbufs_count, &tcp_table->inbufs_total); + if (ret != KNOT_EOK) { + break; + } + if (conn->inbuf.iov_len > 0 && tcp_table->next_ibuf == NULL) { + tcp_table->next_ibuf = conn; + } + } + + // process TCP connection state + switch (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK | + KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_RST)) { + case KNOT_XDP_MSG_SYN: + case (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK): + if (conn == NULL) { + bool synack = (msg->flags & KNOT_XDP_MSG_ACK); + + knot_tcp_table_t *add_table = tcp_table; + if (syn_table != NULL && !synack) { + add_table = syn_table; + if (*tcp_table_lookup(&msg->ip_from, &msg->ip_to, &conn_hash, syn_table) != NULL) { + break; + } + } + + ret = tcp_table_add(msg, conn_hash, add_table, &relay->conn); + if (ret == KNOT_EOK) { + relay->action = synack ? XDP_TCP_ESTABLISH : XDP_TCP_SYN; + if (!(ignore & XDP_TCP_IGNORE_ESTABLISH)) { + relay->auto_answer = synack ? KNOT_XDP_MSG_ACK : (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK); + } + + conn = relay->conn; + conn->state = synack ? XDP_TCP_NORMAL: XDP_TCP_ESTABLISHING; + conn->mss = MAX(msg->mss, 536); // minimal MSS, most importantly not zero! + conn->window_scale = msg->win_scale; + conn_update(conn, msg); + if (!synack) { + conn->acked = dnssec_random_uint32_t(); + conn->ackno = conn->acked; + } + } + } else { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + break; + case KNOT_XDP_MSG_ACK: + if (!seq_ack_match) { + if (syn_table != NULL && msg->payload.iov_len == 0 && + (pconn = tcp_table_lookup(&msg->ip_from, &msg->ip_to, &conn_hash, syn_table)) != NULL && + (conn = *pconn) != NULL && check_seq_ack(msg, conn)) { + // move conn from syn_table to tcp_table + tcp_table_remove(pconn, syn_table); + tcp_table_insert(conn, conn_hash, tcp_table); + relay->conn = conn; + relay->action = XDP_TCP_ESTABLISH; + conn->state = XDP_TCP_NORMAL; + conn_update(conn, msg); + } + } else { + switch (conn->state) { + case XDP_TCP_NORMAL: + case XDP_TCP_CLOSING1: // just a mess, ignore + break; + case XDP_TCP_ESTABLISHING: + conn->state = XDP_TCP_NORMAL; + relay->action = XDP_TCP_ESTABLISH; + break; + case XDP_TCP_CLOSING2: + if (msg->payload.iov_len == 0) { // otherwise ignore close + tcp_table_remove(pconn, tcp_table); + relay->answer = XDP_TCP_FREE; + } + break; + } + } + break; + case (KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK): + if (ignore & XDP_TCP_IGNORE_FIN) { + break; + } + if (!seq_ack_match) { + if (conn != NULL) { + relay->auto_answer = KNOT_XDP_MSG_RST; + relay->auto_seqno = msg->ackno; + } // else ignore. It would be better and possible, but no big value for the price of CPU. + } else { + if (conn->state == XDP_TCP_CLOSING1) { + relay->action = XDP_TCP_CLOSE; + relay->auto_answer = KNOT_XDP_MSG_ACK; + relay->answer = XDP_TCP_FREE; + tcp_table_remove(pconn, tcp_table); + } else if (msg->payload.iov_len == 0) { // otherwise ignore FIN + relay->action = XDP_TCP_CLOSE; + relay->auto_answer = KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK; + conn->state = XDP_TCP_CLOSING2; + } + } + break; + case KNOT_XDP_MSG_RST: + if (conn != NULL && msg->seqno == conn->seqno) { + relay->action = XDP_TCP_RESET; + tcp_table_remove(pconn, tcp_table); + relay->answer = XDP_TCP_FREE; + } else if (conn != NULL) { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + break; + default: + break; + } + + if (!knot_tcp_relay_empty(relay)) { + relay++; + } + } + + return ret; +} + +_public_ +int knot_tcp_reply_data(knot_tcp_relay_t *relay, knot_tcp_table_t *tcp_table, + bool ignore_lastbyte, uint8_t *data, uint32_t len) +{ + if (relay == NULL || tcp_table == NULL || relay->conn == NULL) { + return KNOT_EINVAL; + } + int ret = knot_tcp_outbufs_add(&relay->conn->outbufs, data, len, ignore_lastbyte, + relay->conn->mss, &tcp_table->outbufs_total); + + if (tcp_table->next_obuf == NULL && knot_tcp_outbufs_usage(relay->conn->outbufs) > 0) { + tcp_table->next_obuf = relay->conn; + } + if (tcp_table->next_resend == NULL && knot_tcp_outbufs_usage(relay->conn->outbufs) > 0) { + tcp_table->next_resend = relay->conn; + } + return ret; +} + +static knot_xdp_msg_t *first_msg(knot_xdp_msg_t *msgs, uint32_t n_msgs) +{ + memset(msgs, 0, n_msgs * sizeof(*msgs)); + return msgs - 1; // will be incremented just before first use +} + +static int send_msgs(knot_xdp_msg_t *msgs, uint32_t n_msgs, knot_xdp_socket_t *socket) +{ + assert(socket); + assert(msgs); + + if (n_msgs > 0) { + uint32_t unused; + return knot_xdp_send(socket, msgs, n_msgs, &unused); + } + + return KNOT_EOK; +} + +static void msg_init_from_conn(knot_xdp_msg_t *msg, knot_tcp_conn_t *conn) +{ + memcpy( msg->eth_from, conn->last_eth_loc, sizeof(msg->eth_from)); + memcpy( msg->eth_to, conn->last_eth_rem, sizeof(msg->eth_to)); + memcpy(&msg->ip_from, &conn->ip_loc, sizeof(msg->ip_from)); + memcpy(&msg->ip_to, &conn->ip_rem, sizeof(msg->ip_to)); + + msg->ackno = conn->seqno; + msg->seqno = conn->ackno; + + msg->payload.iov_len = 0; + + msg->win_scale = 14; // maximum possible + msg->win = 0xffff; +} + +static int next_msg(knot_xdp_msg_t *msgs, uint32_t n_msgs, knot_xdp_msg_t **cur, + knot_xdp_socket_t *socket, knot_tcp_relay_t *rl) +{ + (*cur)++; + if (*cur - msgs >= n_msgs) { + int ret = send_msgs(msgs, n_msgs, socket); + if (ret != KNOT_EOK) { + return ret; + } + *cur = first_msg(msgs, n_msgs); + (*cur)++; + } + + knot_xdp_msg_t *msg = *cur; + + knot_xdp_msg_flag_t fl = KNOT_XDP_MSG_TCP; + if (rl->conn->ip_loc.sin6_family == AF_INET6) { + fl |= KNOT_XDP_MSG_IPV6; + } + if (rl->conn->state == XDP_TCP_ESTABLISHING) { + fl |= KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC; + } + + int ret = knot_xdp_send_alloc(socket, fl, msg); + if (ret != KNOT_EOK) { + return ret; + } + + msg_init_from_conn(msg, rl->conn); + + return ret; +} + +_public_ +int knot_tcp_send(knot_xdp_socket_t *socket, knot_tcp_relay_t relays[], + uint32_t relay_count, uint32_t max_at_once) +{ + if (relay_count == 0) { + return KNOT_EOK; + } + if (socket == NULL || relays == NULL) { + return KNOT_EINVAL; + } + + knot_xdp_msg_t msgs[max_at_once], *first = first_msg(msgs, max_at_once), *msg = first; + + for (uint32_t i = 0; i < relay_count; i++) { + knot_tcp_relay_t *rl = &relays[i]; + +#define NEXT_MSG { \ + int ret = next_msg(msgs, max_at_once, &msg, socket, rl); \ + if (ret != KNOT_EOK) { return ret; } \ +} + + if (rl->auto_answer != 0) { + NEXT_MSG + msg->flags |= rl->auto_answer; + if (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_FIN)) { + rl->conn->ackno++; + } + if (rl->auto_answer == KNOT_XDP_MSG_RST) { + msg->seqno = rl->auto_seqno; + } + } + + switch (rl->answer & 0x0f) { + case XDP_TCP_ESTABLISH: + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_SYN; + rl->conn->ackno++; + break; + case XDP_TCP_CLOSE: + NEXT_MSG + msg->flags |= (KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK); + rl->conn->ackno++; + rl->conn->state = XDP_TCP_CLOSING1; + break; + case XDP_TCP_RESET: + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_RST; + break; + case XDP_TCP_NOOP: + default: + break; + } + + size_t can_data = 0; + knot_tcp_outbuf_t *ob; + if (rl->conn != NULL) { + knot_tcp_outbufs_can_send(rl->conn->outbufs, rl->conn->window_size, + rl->answer == XDP_TCP_RESEND, &ob, &can_data); + } + while (can_data > 0) { + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_ACK; + msg->payload.iov_len = ob->len; + memcpy(msg->payload.iov_base, ob->bytes, ob->len); + + if (!ob->sent) { + assert(rl->conn->ackno == msg->seqno); + rl->conn->ackno += msg->payload.iov_len; + } else { + msg->seqno = ob->seqno; + } + + ob->sent = true; + ob->seqno = msg->seqno; + + can_data--; + ob = ob->next; + } +#undef NEXT_MSG + } + + return send_msgs(msgs, msg - first, socket); +} + +static void sweep_reset(knot_tcp_table_t *tcp_table, knot_tcp_relay_t *rl, + ssize_t *free_conns, ssize_t *free_inbuf, ssize_t *free_outbuf, + knot_sweep_stats_t *stats, knot_sweep_counter_t counter) +{ + rl->answer = XDP_TCP_RESET | XDP_TCP_FREE; + tcp_table_remove(tcp_table_re_lookup(rl->conn, tcp_table), tcp_table); // also updates tcp_table->next_* + + *free_conns -= 1; + *free_inbuf -= rl->conn->inbuf.iov_len; + *free_outbuf -= knot_tcp_outbufs_usage(rl->conn->outbufs); + + knot_sweep_stats_incr(stats, counter); +} + +_public_ +int knot_tcp_sweep(knot_tcp_table_t *tcp_table, + uint32_t close_timeout, uint32_t reset_timeout, + uint32_t resend_timeout, uint32_t limit_conn_count, + size_t limit_ibuf_size, size_t limit_obuf_size, + knot_tcp_relay_t *relays, uint32_t max_relays, + struct knot_sweep_stats *stats) +{ + if (tcp_table == NULL || relays == NULL || max_relays < 1) { + return KNOT_EINVAL; + } + + uint32_t now = get_timestamp(); + memset(relays, 0, max_relays * sizeof(*relays)); + knot_tcp_relay_t *rl = relays, *rl_max = rl + max_relays; + + ssize_t free_conns = (ssize_t)(tcp_table->usage - limit_conn_count); + ssize_t free_inbuf = (ssize_t)(tcp_table->inbufs_total - MIN(limit_ibuf_size, SSIZE_MAX)); + ssize_t free_outbuf = (ssize_t)(tcp_table->outbufs_total - MIN(limit_obuf_size, SSIZE_MAX)); + + // reset connections to free ibufs + while (free_inbuf > 0 && rl != rl_max) { + if (tcp_table->next_ibuf->inbuf.iov_len == 0) { // this conn might have get rid of ibuf in the meantime + next_ptr_ibuf(&tcp_table->next_ibuf); + } + assert(tcp_table->next_ibuf != NULL); + rl->conn = tcp_table->next_ibuf; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, KNOT_SWEEP_CTR_LIMIT_IBUF); + rl++; + } + + // reset connections to free obufs + while (free_outbuf > 0 && rl != rl_max) { + if (knot_tcp_outbufs_usage(tcp_table->next_obuf->outbufs) == 0) { + next_ptr_obuf(&tcp_table->next_obuf); + } + assert(tcp_table->next_obuf != NULL); + rl->conn = tcp_table->next_obuf; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, KNOT_SWEEP_CTR_LIMIT_OBUF); + rl++; + } + + // reset connections to free their count, and old ones + knot_tcp_conn_t *conn, *next; + WALK_LIST_DELSAFE(conn, next, *tcp_table_timeout(tcp_table)) { + if ((free_conns <= 0 && now - conn->last_active < reset_timeout) || rl == rl_max) { + break; + } + + rl->conn = conn; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, KNOT_SWEEP_CTR_LIMIT_CONN); + rl++; + } + + // close old connections + while (tcp_table->next_close != NULL && + now - tcp_table->next_close->last_active >= close_timeout && + rl != rl_max) { + if (tcp_table->next_close->state != XDP_TCP_CLOSING1) { + rl->conn = tcp_table->next_close; + rl->answer = XDP_TCP_CLOSE; + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_TIMEOUT); + rl++; + } + next_node_ptr(&tcp_table->next_close); + } + + // resend unACKed data + while (tcp_table->next_resend != NULL && + now - tcp_table->next_resend->last_active >= resend_timeout && + rl != rl_max) { + rl->conn = tcp_table->next_resend; + rl->answer = XDP_TCP_RESEND; + rl++; + next_ptr_obuf(&tcp_table->next_resend); + } + + return KNOT_EOK; +} + +_public_ +void knot_tcp_cleanup(knot_tcp_table_t *tcp_table, knot_tcp_relay_t relays[], + uint32_t relay_count) +{ + (void)tcp_table; + for (uint32_t i = 0; i < relay_count; i++) { + if (relays[i].answer & XDP_TCP_FREE) { + del_conn(relays[i].conn); + } + free(relays[i].inbufs); + } +} diff --git a/src/libknot/xdp/tcp.h b/src/libknot/xdp/tcp.h new file mode 100644 index 0000000..da0f1a4 --- /dev/null +++ b/src/libknot/xdp/tcp.h @@ -0,0 +1,227 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief TCP over XDP IO interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include "libknot/xdp/msg.h" +#include "libknot/xdp/xdp.h" + +struct knot_sweep_stats; + +typedef enum { + XDP_TCP_NOOP = 0, + XDP_TCP_SYN = 1, + XDP_TCP_ESTABLISH = 2, + XDP_TCP_CLOSE = 3, + XDP_TCP_RESET = 4, + XDP_TCP_RESEND = 5, + + XDP_TCP_FREE = 0x10, +} knot_tcp_action_t; + +typedef enum { + XDP_TCP_NORMAL, + XDP_TCP_ESTABLISHING, + XDP_TCP_CLOSING1, // FIN+ACK sent + XDP_TCP_CLOSING2, // FIN+ACK received and sent +} knot_tcp_state_t; + +typedef enum { + XDP_TCP_FREE_NONE, + XDP_TCP_FREE_DATA, + XDP_TCP_FREE_PREFIX, +} knot_tcp_relay_free_t; + +typedef enum { + XDP_TCP_IGNORE_NONE = 0, + XDP_TCP_IGNORE_ESTABLISH = (1 << 0), + XDP_TCP_IGNORE_DATA_ACK = (1 << 1), + XDP_TCP_IGNORE_FIN = (1 << 2), +} knot_tcp_ignore_t; + +typedef struct knot_tcp_conn { + struct { + struct knot_tcp_conn *list_node_next; + struct knot_tcp_conn *list_node_prev; + } list_node_placeholder; + struct sockaddr_in6 ip_rem; + struct sockaddr_in6 ip_loc; + uint8_t last_eth_rem[ETH_ALEN]; + uint8_t last_eth_loc[ETH_ALEN]; + uint16_t mss; + uint8_t window_scale; + uint32_t seqno; + uint32_t ackno; + uint32_t acked; + uint32_t window_size; + uint32_t last_active; + uint32_t establish_rtt; // in microseconds + knot_tcp_state_t state; + struct iovec inbuf; + struct knot_tcp_outbuf *outbufs; + struct knot_tcp_conn *next; +} knot_tcp_conn_t; + +typedef struct { + size_t size; + size_t usage; + size_t inbufs_total; + size_t outbufs_total; + uint64_t hash_secret[2]; + knot_tcp_conn_t *next_close; + knot_tcp_conn_t *next_ibuf; + knot_tcp_conn_t *next_obuf; + knot_tcp_conn_t *next_resend; + knot_tcp_conn_t *conns[]; +} knot_tcp_table_t; + +typedef struct { + const knot_xdp_msg_t *msg; + knot_tcp_action_t action; + knot_xdp_msg_flag_t auto_answer; + uint32_t auto_seqno; + knot_tcp_action_t answer; + struct iovec *inbufs; + size_t inbufs_count; + knot_tcp_conn_t *conn; +} knot_tcp_relay_t; + +/*! + * \brief Return next TCP sequence number. + */ +inline static uint32_t knot_tcp_next_seqno(const knot_xdp_msg_t *msg) +{ + uint32_t res = msg->seqno + msg->payload.iov_len; + if (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_FIN)) { + res++; + } + return res; +} + +/*! + * \brief Check if the relay is empty. + */ +inline static bool knot_tcp_relay_empty(const knot_tcp_relay_t *relay) +{ + return relay->action == XDP_TCP_NOOP && relay->answer == XDP_TCP_NOOP && + relay->auto_answer == 0 && relay->inbufs_count == 0; +} + +/*! + * \brief Allocate TCP connection-handling hash table. + * + * \param size Number of records for the hash table. + * \param secret_share Optional: share the hashing secret with another table. + * + * \note Hashing conflicts are solved by single-linked-lists in each record. + * + * \return The table, or NULL. + */ +knot_tcp_table_t *knot_tcp_table_new(size_t size, knot_tcp_table_t *secret_share); + +/*! + * \brief Free TCP connection hash table including all connection records. + * + * \note The freed connections are not closed nor reset. + */ +void knot_tcp_table_free(knot_tcp_table_t *table); + +/*! + * \brief Process received packets, prepare automatic responses (e.g. ACK), pick incoming data. + * + * \param relays Out: relays to be filled with message/connection details. + * \param msgs Packets received by knot_xdp_recv(). + * \param msg_count Number of received packets. + * \param tcp_table Table of TCP connections. + * \param syn_table Optional: extra table for handling partially established connections. + * \param ignore Ignore specific TCP packets indication. + * + * \return KNOT_E* + */ +int knot_tcp_recv(knot_tcp_relay_t *relays, knot_xdp_msg_t msgs[], uint32_t msg_count, + knot_tcp_table_t *tcp_table, knot_tcp_table_t *syn_table, + knot_tcp_ignore_t ignore); + +/*! + * \brief Prepare data (payload) to be sent as a response on specific relay. + * + * \param relay Relay with active connection. + * \param tcp_table TCP table. + * \param ignore_lastbyte Evil mode: drop last byte of the payload. + * \param data Data payload, possibly > MSS and > window. + * \param len Payload length, < 64k. + * + * \return KNOT_E* + */ +int knot_tcp_reply_data(knot_tcp_relay_t *relay, knot_tcp_table_t *tcp_table, + bool ignore_lastbyte, uint8_t *data, uint32_t len); + +/*! + * \brief Send TCP packets. + * + * \param socket XDP socket to send through. + * \param relays Connection changes and data. + * \param relay_count Number of connection changes and data. + * \param max_at_once Limit of packet batch sent by knot_xdp_send(). + * + * \return KNOT_E* + */ +int knot_tcp_send(knot_xdp_socket_t *socket, knot_tcp_relay_t relays[], + uint32_t relay_count, uint32_t max_at_once); + +/*! + * \brief Cleanup old TCP connections, perform timeout checks. + * + * \param tcp_table TCP connection table to clean up. + * \param close_timeout Gracefully close connections older than this (usecs). + * \param reset_timeout Reset connections older than this (usecs). + * \param resend_timeout Resend unAcked data older than this (usecs). + * \param limit_conn_count Limit of active connections in TCP table, reset if more. + * \param limit_ibuf_size Limit of memory usage by input buffers, reset if exceeded. + * \param limit_obuf_size Limit of memory usage by output buffers, reset if exceeded. + * \param relays Out: relays to be filled with close/reset instructions for knot_tcp_send(). + * \param max_relays Maximum relays to be used. + * \param stats Out: sweeped out connection statistics. + * + * \return KNOT_E* + */ +int knot_tcp_sweep(knot_tcp_table_t *tcp_table, + uint32_t close_timeout, uint32_t reset_timeout, + uint32_t resend_timeout, uint32_t limit_conn_count, + size_t limit_ibuf_size, size_t limit_obuf_size, + knot_tcp_relay_t *relays, uint32_t max_relays, + struct knot_sweep_stats *stats); + +/*! + * \brief Free resources of closed/reset connections. + * + * \param tcp_table TCP table with connections. + * \param relays Relays with closed/reset (or other, ignored) connections. + * \param relay_count Number of relays. + */ +void knot_tcp_cleanup(knot_tcp_table_t *tcp_table, knot_tcp_relay_t relays[], + uint32_t relay_count); + +/*! @} */ diff --git a/src/libknot/xdp/tcp_iobuf.c b/src/libknot/xdp/tcp_iobuf.c new file mode 100644 index 0000000..88ebcc3 --- /dev/null +++ b/src/libknot/xdp/tcp_iobuf.c @@ -0,0 +1,266 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#include "libknot/xdp/tcp_iobuf.h" + +#include "contrib/macros.h" +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/error.h" + +static void iov_clear(struct iovec *iov) +{ + free(iov->iov_base); + memset(iov, 0, sizeof(*iov)); +} + +static void iov_inc(struct iovec *iov, size_t shift) +{ + assert(shift <= iov->iov_len); + iov->iov_base += shift; + iov->iov_len -= shift; +} + +/*! \brief Strip 2-byte length prefix from a payload. */ +static void iov_inc2(struct iovec *iov) +{ + iov_inc(iov, sizeof(uint16_t)); +} + +static size_t tcp_payload_len(const struct iovec *payload) +{ + assert(payload->iov_len >= 2); + uint16_t val; + memcpy(&val, payload->iov_base, sizeof(val)); + return be16toh(val) + sizeof(val); +} + +static bool iov_inc_pf(struct iovec *iov) +{ + size_t shift = tcp_payload_len(iov); + if (iov->iov_len >= shift) { + iov_inc(iov, shift); + return true; + } else { + return false; + } +} + +static size_t iov_count(const struct iovec *iov) +{ + size_t res = 0; + struct iovec tmp = *iov; + while (tmp.iov_len >= sizeof(uint16_t) && iov_inc_pf(&tmp)) { + res++; + } + return res; +} + +static void iov_append(struct iovec *what, const struct iovec *with) +{ + // NOTE: what->iov_base must be pre-allocated large enough + memcpy(what->iov_base + what->iov_len, with->iov_base, with->iov_len); + what->iov_len += with->iov_len; +} + +_public_ +int knot_tcp_inbuf_update(struct iovec *buffer, struct iovec data, + struct iovec **inbufs, size_t *inbufs_count, + size_t *buffers_total) +{ + size_t res_count = 0; + struct iovec *res = NULL, *cur = NULL; + + *inbufs = NULL; + *inbufs_count = 0; + + if (data.iov_len < 1) { + return KNOT_EOK; + } + if (buffer->iov_len == 1) { + ((uint8_t *)buffer->iov_base)[1] = ((uint8_t *)data.iov_base)[0]; + buffer->iov_len++; + iov_inc(&data, 1); + if (data.iov_len < 1) { + return KNOT_EOK; + } + } + if (buffer->iov_len > 0) { + size_t buffer_req = tcp_payload_len(buffer); + assert(buffer_req > buffer->iov_len); + struct iovec data_use = { data.iov_base, buffer_req - buffer->iov_len }; + if (data_use.iov_len <= data.iov_len) { // usable payload combined from buffer and data ---> res[0] allocated tohether with res + iov_inc(&data, data_use.iov_len); + + res_count = 1 + iov_count(&data); + res = malloc(res_count * sizeof(*res) + buffer_req); + if (res == NULL) { + return KNOT_ENOMEM; + } + res[0].iov_base = (void *)(res + res_count); + res[0].iov_len = 0; + iov_append(&res[0], buffer); + iov_append(&res[0], &data_use); + assert(res[0].iov_len == buffer_req); + iov_inc2(&res[0]); + + cur = &res[1]; + *buffers_total -= buffer->iov_len; + iov_clear(buffer); + } else { // just extend the buffer with data + void *bufnew = realloc(buffer->iov_base, buffer->iov_len + data.iov_len); + if (bufnew == NULL) { + return KNOT_ENOMEM; + } + buffer->iov_base = bufnew; + iov_append(buffer, &data); + *buffers_total += data.iov_len; + return KNOT_EOK; + } + } else { // just allocate res + res_count = iov_count(&data); + if (res_count > 0) { + res = malloc(res_count * sizeof(*res)); + if (res == NULL) { + return KNOT_ENOMEM; + } + cur = &res[0]; + } + } + + void *last; + while (data.iov_len > 1) { + last = data.iov_base; + if (!iov_inc_pf(&data)) { + break; + } + assert(cur); + cur->iov_base = last; + cur->iov_len = data.iov_base - last; + iov_inc2(cur); + cur++; + } + assert(cur == ((res_count) ? res + res_count : res)); + + // store the final incomplete payload to buffer + if (data.iov_len > 0) { + assert(buffer->iov_base == NULL); + buffer->iov_base = malloc(MAX(data.iov_len, 2)); + if (buffer->iov_base == NULL) { + free(res); + return KNOT_ENOMEM; + } + *buffers_total += MAX(data.iov_len, 2); + buffer->iov_len = 0; + iov_append(buffer, &data); + } + + *inbufs = res; + *inbufs_count = res_count; + + return KNOT_EOK; +} + +_public_ +int knot_tcp_outbufs_add(knot_tcp_outbuf_t **bufs, uint8_t *data, size_t len, + bool ignore_lastbyte, uint32_t mss, size_t *outbufs_total) +{ + if (len > UINT16_MAX) { + return KNOT_ELIMIT; + } + knot_tcp_outbuf_t **end = bufs; + while (*end != NULL) { // NOTE: this can be optimized by adding "end" pointer for the price of larger knot_tcp_conn_t struct + end = &(*end)->next; + } + uint16_t prefix = htobe16(len), prefix_len = sizeof(prefix); + while (len > 0) { + uint16_t newlen = MIN(len + prefix_len, mss); + knot_tcp_outbuf_t *newob = calloc(1, sizeof(*newob) + newlen); + if (newob == NULL) { + return KNOT_ENOMEM; + } + *outbufs_total += sizeof(*newob) + newlen; + newob->len = newlen; + if (ignore_lastbyte) { + newob->len--; + } + memcpy(newob->bytes, &prefix, prefix_len); + memcpy(newob->bytes + prefix_len, data, newlen - prefix_len); + + *end = newob; + end = &newob->next; + + data += newlen - prefix_len; + len -= newlen - prefix_len; + + prefix_len = 0; + } + return KNOT_EOK; +} + +static bool seqno_lower(uint32_t seqno, uint32_t ackno, uint32_t ackno_min) +{ + if (ackno_min <= ackno) { + return (seqno >= ackno_min && seqno <= ackno); + } else { + return (seqno >= ackno_min || seqno <= ackno); + } +} + +_public_ +void knot_tcp_outbufs_ack(knot_tcp_outbuf_t **bufs, uint32_t ackno, size_t *outbufs_total) +{ + uint32_t ackno_min = ackno - (UINT32_MAX / 2); // FIXME better? + while (*bufs != NULL && (*bufs)->sent && seqno_lower((*bufs)->seqno + (*bufs)->len, ackno, ackno_min)) { + knot_tcp_outbuf_t *tofree = *bufs; + *bufs = tofree->next; + *outbufs_total -= tofree->len + sizeof(*tofree); + free(tofree); + } +} + +_public_ +void knot_tcp_outbufs_can_send(knot_tcp_outbuf_t *bufs, ssize_t window_size, bool resend, + knot_tcp_outbuf_t **send_start, size_t *send_count) +{ + *send_count = 0; + *send_start = bufs; + while (*send_start != NULL && (*send_start)->sent && !resend) { + window_size -= (*send_start)->len; + *send_start = (*send_start)->next; + } + + knot_tcp_outbuf_t *can_send = *send_start; + while (can_send != NULL && window_size >= can_send->len) { + (*send_count)++; + window_size -= can_send->len; + can_send = can_send->next; + } +} + +_public_ +size_t knot_tcp_outbufs_usage(knot_tcp_outbuf_t *bufs) +{ + size_t res = 0; + for (knot_tcp_outbuf_t *i = bufs; i != NULL; i = i->next) { + res += i->len + sizeof(*i); + } + return res; +} diff --git a/src/libknot/xdp/tcp_iobuf.h b/src/libknot/xdp/tcp_iobuf.h new file mode 100644 index 0000000..a27eeab --- /dev/null +++ b/src/libknot/xdp/tcp_iobuf.h @@ -0,0 +1,120 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief TCP buffer helpers. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <sys/uio.h> + +typedef struct knot_tcp_outbuf { + struct knot_tcp_outbuf *next; + uint32_t len; + uint32_t seqno; + bool sent; + uint8_t bytes[]; +} knot_tcp_outbuf_t; + +typedef enum { + KNOT_SWEEP_CTR_TIMEOUT = 0, + KNOT_SWEEP_CTR_LIMIT_CONN = 1, + KNOT_SWEEP_CTR_LIMIT_IBUF = 2, + KNOT_SWEEP_CTR_LIMIT_OBUF = 3, +} knot_sweep_counter_t; + +typedef struct knot_sweep_stats { + uint32_t total; + uint32_t counters[4]; +} knot_sweep_stats_t; + +inline static void knot_sweep_stats_incr(knot_sweep_stats_t *stats, knot_sweep_counter_t counter) +{ + (stats->counters[counter])++; + (stats->total)++; +} + +inline static void knot_sweep_stats_reset(knot_sweep_stats_t *stats) +{ + memset(stats, 0, sizeof(*stats)); +} + +/*! + * \brief Handle DNS-over-TCP payloads in buffer and message. + * + * \param buffer In/out: persistent buffer to store incomplete DNS payloads between receiving packets. + * \param data In: momental DNS payloads in incoming packet. + * \param inbufs Out: list of incoming DNS messages. + * \param inbufs_count Out: number of inbufs. + * \param buffers_total In/Out: total size of buffers (will be increased or decreased). + * + * \return KNOT_EOK, KNOT_ENOMEM + */ +int knot_tcp_inbuf_update(struct iovec *buffer, struct iovec data, + struct iovec **inbufs, size_t *inbufs_count, + size_t *buffers_total); + +/*! + * \brief Add payload to be sent by TCP, to output buffers. + * + * \param bufs Output buffers to be updated. + * \param data Payload to be sent. + * \param len Payload length. + * \param ignore_lastbyte Evil mode: drop last byte of the payload. + * \param mss Connection outgoing MSS. + * \param outbufs_total In/out: total outbuf statistic to be updated. + * + * \return KNOT_E* + */ +int knot_tcp_outbufs_add(knot_tcp_outbuf_t **bufs, uint8_t *data, size_t len, + bool ignore_lastbyte, uint32_t mss, size_t *outbufs_total); + +/*! + * \brief Remove+free acked data from output buffers. + * + * \param bufs Output buffers to be updated. + * \param ackno Ackno of received ACK. + * \param outbufs_total In/out: total outbuf statistic to be updated. + */ +void knot_tcp_outbufs_ack(knot_tcp_outbuf_t **bufs, uint32_t ackno, size_t *outbufs_total); + +/*! + * \brief Prepare output buffers to be sent now. + * + * \param bufs Output buffers to be updated. + * \param window_size Connection outgoing window size. + * \param resend Send also possibly already sent data. + * \param send_start Out: first output buffer to be sent. + * \param send_count Out: number of output buffers to be sent. + */ +void knot_tcp_outbufs_can_send(knot_tcp_outbuf_t *bufs, ssize_t window_size, bool resend, + knot_tcp_outbuf_t **send_start, size_t *send_count); + +/*! + * \brief Compute allocated size of output buffers. + */ +size_t knot_tcp_outbufs_usage(knot_tcp_outbuf_t *bufs); + +/*! @} */ diff --git a/src/libknot/xdp/xdp.c b/src/libknot/xdp/xdp.c new file mode 100644 index 0000000..975276d --- /dev/null +++ b/src/libknot/xdp/xdp.c @@ -0,0 +1,568 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <errno.h> +#include <netinet/in.h> +#include <linux/if_ether.h> +#include <linux/udp.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/errcode.h" +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/bpf-user.h" +#include "libknot/xdp/eth.h" +#include "libknot/xdp/msg_init.h" +#include "libknot/xdp/protocols.h" +#include "libknot/xdp/xdp.h" +#include "contrib/macros.h" +#include "contrib/net.h" + +#define FRAME_SIZE 2048 + +#define FRAME_COUNT_TX 2048 +#define FRAME_COUNT_RX 2048 +#define FRAME_COUNT (FRAME_COUNT_TX + FRAME_COUNT_RX) + +#define RING_LEN_TX FRAME_COUNT_TX +#define RING_LEN_CQ FRAME_COUNT_TX +#define RING_LEN_RX FRAME_COUNT_RX +/* It's recommended that the FQ ring size >= HW RX ring size + AF_XDP RX ring size. */ +#define RING_LEN_FQ 8192 + +#define ALLOC_RETRY_NUM 15 +#define ALLOC_RETRY_DELAY 20 // In nanoseconds. + +/* With recent compilers we statically check #defines for settings that + * get refused by AF_XDP drivers (in current versions, at least). */ +#if (__STDC_VERSION__ >= 201112L) +#define IS_POWER_OF_2(n) (((n) & (n - 1)) == 0) +_Static_assert((FRAME_SIZE == 4096 || FRAME_SIZE == 2048) + && IS_POWER_OF_2(RING_LEN_TX) && IS_POWER_OF_2(RING_LEN_RX) + && IS_POWER_OF_2(RING_LEN_CQ) && IS_POWER_OF_2(RING_LEN_FQ) + && FRAME_COUNT_TX <= (1 << 16) /* see tx_free_indices */ + , "Incorrect #define combination for AF_XDP."); +#endif + +struct umem_frame { + uint8_t bytes[FRAME_SIZE]; +}; + +static int configure_xsk_umem(struct kxsk_umem **out_umem) +{ + /* Allocate memory and call driver to create the UMEM. */ + struct kxsk_umem *umem = calloc(1, + offsetof(struct kxsk_umem, tx_free_indices) + + sizeof(umem->tx_free_indices[0]) * FRAME_COUNT_TX); + if (umem == NULL) { + return KNOT_ENOMEM; + } + + int ret = posix_memalign((void **)&umem->frames, getpagesize(), + FRAME_SIZE * FRAME_COUNT); + if (ret != 0) { + free(umem); + return KNOT_ENOMEM; + } + + const struct xsk_umem_config config = { + .fill_size = RING_LEN_FQ, + .comp_size = RING_LEN_CQ, + .frame_size = FRAME_SIZE, + .frame_headroom = KNOT_XDP_PKT_ALIGNMENT, + }; + + ret = xsk_umem__create(&umem->umem, umem->frames, FRAME_SIZE * FRAME_COUNT, + &umem->fq, &umem->cq, &config); + if (ret != KNOT_EOK) { + free(umem->frames); + free(umem); + return ret; + } + *out_umem = umem; + + /* Designate the starting chunk of buffers for TX, and put them onto the stack. */ + umem->tx_free_count = FRAME_COUNT_TX; + for (uint32_t i = 0; i < FRAME_COUNT_TX; ++i) { + umem->tx_free_indices[i] = i; + } + + /* Designate the rest of buffers for RX, and pass them to the driver. */ + uint32_t idx = 0; + ret = xsk_ring_prod__reserve(&umem->fq, FRAME_COUNT_RX, &idx); + if (ret != FRAME_COUNT_RX) { + assert(0); + return KNOT_ERROR; + } + assert(idx == 0); + assert(FRAME_COUNT == FRAME_COUNT_TX + FRAME_COUNT_RX); + for (uint32_t i = FRAME_COUNT_TX; i < FRAME_COUNT; ++i) { + *xsk_ring_prod__fill_addr(&umem->fq, idx++) = i * FRAME_SIZE; + } + xsk_ring_prod__submit(&umem->fq, FRAME_COUNT_RX); + + return KNOT_EOK; +} + +static void deconfigure_xsk_umem(struct kxsk_umem *umem) +{ + (void)xsk_umem__delete(umem->umem); + free(umem->frames); + free(umem); +} + +static int configure_xsk_socket(struct kxsk_umem *umem, + const struct kxsk_iface *iface, + knot_xdp_socket_t **out_sock) +{ + knot_xdp_socket_t *xsk_info = calloc(1, sizeof(*xsk_info)); + if (xsk_info == NULL) { + return KNOT_ENOMEM; + } + xsk_info->iface = iface; + xsk_info->umem = umem; + + const struct xsk_socket_config sock_conf = { + .tx_size = RING_LEN_TX, + .rx_size = RING_LEN_RX, + .libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD, + }; + + int ret = xsk_socket__create(&xsk_info->xsk, iface->if_name, + iface->if_queue, umem->umem, + &xsk_info->rx, &xsk_info->tx, &sock_conf); + if (ret != 0) { + free(xsk_info); + return ret; + } + + *out_sock = xsk_info; + return KNOT_EOK; +} + +_public_ +int knot_xdp_init(knot_xdp_socket_t **socket, const char *if_name, int if_queue, + knot_xdp_filter_flag_t flags, uint16_t udp_port, uint16_t quic_port, + knot_xdp_load_bpf_t load_bpf, const void *xdp_config) +{ + if (socket == NULL || if_name == NULL || + (udp_port == quic_port && (flags & KNOT_XDP_FILTER_UDP) && (flags & KNOT_XDP_FILTER_QUIC)) || + (flags & (KNOT_XDP_FILTER_UDP | KNOT_XDP_FILTER_TCP | KNOT_XDP_FILTER_QUIC)) == 0) { + return KNOT_EINVAL; + } + + struct kxsk_iface *iface; + int ret = kxsk_iface_new(if_name, if_queue, load_bpf, &iface); + if (ret != KNOT_EOK) { + return ret; + } + + /* Initialize shared packet_buffer for umem usage. */ + struct kxsk_umem *umem = NULL; + ret = configure_xsk_umem(&umem); + if (ret != KNOT_EOK) { + kxsk_iface_free(iface); + return ret; + } + + ret = configure_xsk_socket(umem, iface, socket); + if (ret != KNOT_EOK) { + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + return ret; + } + + (*socket)->frame_limit = FRAME_SIZE; + ret = knot_eth_mtu(if_name); + if (ret > 0) { + (*socket)->frame_limit = MIN((unsigned)ret, (*socket)->frame_limit); + } + + if (flags & KNOT_XDP_FILTER_ROUTE) { + ret = knot_eth_vlans(&(*socket)->vlan_map, &(*socket)->vlan_map_max); + if (ret != KNOT_EOK) { + xsk_socket__delete((*socket)->xsk); + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + free(*socket); + *socket = NULL; + return ret; + } + } + + ret = kxsk_socket_start(iface, flags, udp_port, quic_port, (*socket)->xsk); + if (ret != KNOT_EOK) { + free((*socket)->vlan_map); + xsk_socket__delete((*socket)->xsk); + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + free(*socket); + *socket = NULL; + return ret; + } + + return ret; +} + +_public_ +void knot_xdp_deinit(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return; + } + if (unlikely(socket->send_mock != NULL)) { + free(socket); + return; + } + + kxsk_socket_stop(socket->iface); + xsk_socket__delete(socket->xsk); + deconfigure_xsk_umem(socket->umem); + + kxsk_iface_free((struct kxsk_iface *)/*const-cast*/socket->iface); + free(socket->vlan_map); + free(socket); +} + +_public_ +int knot_xdp_socket_fd(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return 0; + } + + return xsk_socket__fd(socket->xsk); +} + +static void tx_free_relative(struct kxsk_umem *umem, uint64_t addr_relative) +{ + /* The address may not point to *start* of buffer, but `/` solves that. */ + uint64_t index = addr_relative / FRAME_SIZE; + assert(index < FRAME_COUNT); + umem->tx_free_indices[umem->tx_free_count++] = index; +} + +_public_ +void knot_xdp_send_prepare(knot_xdp_socket_t *socket) +{ + if (socket == NULL || unlikely(socket->send_mock != NULL)) { + return; + } + + struct kxsk_umem *const umem = socket->umem; + struct xsk_ring_cons *const cq = &umem->cq; + + uint32_t idx = 0; + const uint32_t completed = xsk_ring_cons__peek(cq, UINT32_MAX, &idx); + if (completed == 0) { + return; + } + assert(umem->tx_free_count + completed <= FRAME_COUNT_TX); + + for (uint32_t i = 0; i < completed; ++i) { + uint64_t addr_relative = *xsk_ring_cons__comp_addr(cq, idx++); + tx_free_relative(umem, addr_relative); + } + + xsk_ring_cons__release(cq, completed); +} + +static struct umem_frame *alloc_tx_frame(knot_xdp_socket_t *socket) +{ + if (unlikely(socket->send_mock != NULL)) { + return malloc(sizeof(struct umem_frame)); + } + + const struct timespec delay = { .tv_nsec = ALLOC_RETRY_DELAY }; + struct kxsk_umem *umem = socket->umem; + + for (int i = 0; unlikely(umem->tx_free_count == 0); i++) { + if (i == ALLOC_RETRY_NUM) { + return NULL; + } + nanosleep(&delay, NULL); + knot_xdp_send_prepare(socket); + } + + uint32_t index = umem->tx_free_indices[--umem->tx_free_count]; + return umem->frames + index; +} + +static void prepare_payload(knot_xdp_msg_t *msg, void *uframe) +{ + size_t hdr_len = prot_write_hdrs_len(msg); + msg->payload.iov_base = uframe + hdr_len + KNOT_XDP_PKT_ALIGNMENT; + msg->payload.iov_len = FRAME_SIZE - hdr_len - KNOT_XDP_PKT_ALIGNMENT; +} + +_public_ +int knot_xdp_send_alloc(knot_xdp_socket_t *socket, knot_xdp_msg_flag_t flags, + knot_xdp_msg_t *out) +{ + if (socket == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct umem_frame *uframe = alloc_tx_frame(socket); + if (uframe == NULL) { + return KNOT_ENOMEM; + } + + msg_init(out, flags); + prepare_payload(out, uframe); + + return KNOT_EOK; +} + +_public_ +int knot_xdp_reply_alloc(knot_xdp_socket_t *socket, const knot_xdp_msg_t *query, + knot_xdp_msg_t *out) +{ + if (socket == NULL || query == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct umem_frame *uframe = alloc_tx_frame(socket); + if (uframe == NULL) { + return KNOT_ENOMEM; + } + + msg_init_reply(out, query); + prepare_payload(out, uframe); + + return KNOT_EOK; +} + +static void free_unsent(knot_xdp_socket_t *socket, const knot_xdp_msg_t *msg) +{ + if (unlikely(socket->send_mock != NULL)) { + free(msg->payload.iov_base - prot_write_hdrs_len(msg) - KNOT_XDP_PKT_ALIGNMENT); + return; + } + uint64_t addr_relative = (uint8_t *)msg->payload.iov_base + - socket->umem->frames->bytes; + tx_free_relative(socket->umem, addr_relative); +} + +_public_ +int knot_xdp_send(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count, uint32_t *sent) +{ + if (socket == NULL || msgs == NULL || sent == NULL) { + return KNOT_EINVAL; + } + if (unlikely(socket->send_mock != NULL)) { + int ret = socket->send_mock(socket, msgs, count, sent); + for (uint32_t i = 0; i < count; ++i) { + free_unsent(socket, &msgs[i]); + } + return ret; + } + + /* Now we want to do something close to + * xsk_ring_prod__reserve(&socket->tx, count, *idx) + * but we don't know in advance if we utilize *whole* `count`, + * and the API doesn't allow "cancelling reservations". + * Therefore we handle `socket->tx.cached_prod` by hand. + */ + if (xsk_prod_nb_free(&socket->tx, count) < count) { + /* This situation was sometimes observed in the emulated XDP mode. */ + for (uint32_t i = 0; i < count; ++i) { + free_unsent(socket, &msgs[i]); + } + return KNOT_ENOBUFS; + } + uint32_t idx = socket->tx.cached_prod; + + for (uint32_t i = 0; i < count; ++i) { + const knot_xdp_msg_t *msg = &msgs[i]; + + if (empty_msg(msg)) { + free_unsent(socket, msg); + } else { + size_t hdr_len = prot_write_hdrs_len(msg); + size_t tot_len = hdr_len + msg->payload.iov_len; + uint8_t *msg_beg = msg->payload.iov_base - hdr_len; + uint16_t mss = MIN(socket->frame_limit - hdr_len, KNOT_TCP_MSS); + prot_write_eth(msg_beg, msg, msg_beg + tot_len, mss); + + *xsk_ring_prod__tx_desc(&socket->tx, idx++) = (struct xdp_desc) { + .addr = msg_beg - socket->umem->frames->bytes, + .len = tot_len, + }; + } + } + + *sent = idx - socket->tx.cached_prod; + assert(*sent <= count); + socket->tx.cached_prod = idx; + xsk_ring_prod__submit(&socket->tx, *sent); + socket->kernel_needs_wakeup = true; + + return KNOT_EOK; +} + +_public_ +void knot_xdp_send_free(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count) +{ + for (uint32_t i = 0; i < count; i++) { + free_unsent(socket, &msgs[i]); + } +} + +_public_ +int knot_xdp_send_finish(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return KNOT_EINVAL; + } + + /* Trigger sending queued packets. */ + if (!socket->kernel_needs_wakeup) { + return KNOT_EOK; + } + + int ret = sendto(xsk_socket__fd(socket->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + const bool is_ok = (ret >= 0); + // List of "safe" errors taken from + // https://github.com/torvalds/linux/blame/master/samples/bpf/xdpsock_user.c + const bool is_again = !is_ok && (errno == ENOBUFS || errno == EAGAIN + || errno == EBUSY || errno == ENETDOWN); + // Some of the !is_ok cases are a little unclear - what to do about the syscall, + // including how caller of _sendmsg_finish() should react. + if (is_ok || !is_again) { + socket->kernel_needs_wakeup = false; + } + if (is_again) { + return KNOT_EAGAIN; + } else if (is_ok) { + return KNOT_EOK; + } else { + return -errno; + } + /* This syscall might be avoided with a newer kernel feature (>= 5.4): + https://www.kernel.org/doc/html/latest/networking/af_xdp.html#xdp-use-need-wakeup-bind-flag + Unfortunately it's not easy to continue supporting older kernels + when using this feature on newer ones. + */ +} + +_public_ +int knot_xdp_recv(knot_xdp_socket_t *socket, knot_xdp_msg_t msgs[], + uint32_t max_count, uint32_t *count, size_t *wire_size) +{ + if (socket == NULL || msgs == NULL || count == NULL) { + return KNOT_EINVAL; + } + + uint32_t idx = 0; + const uint32_t available = xsk_ring_cons__peek(&socket->rx, max_count, &idx); + if (available == 0) { + *count = 0; + return KNOT_EOK; + } + assert(available <= max_count); + + for (uint32_t i = 0; i < available; ++i) { + knot_xdp_msg_t *msg = &msgs[i]; + const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&socket->rx, idx++); + uint8_t *uframe_p = (uint8_t *)socket->umem->frames + desc->addr; + + void *payl_end; + void *payl_start = prot_read_eth(uframe_p, msg, &payl_end, + socket->vlan_map, socket->vlan_map_max); + + msg->payload.iov_base = payl_start; + msg->payload.iov_len = payl_end - payl_start; + msg->mss = MIN(msg->mss, FRAME_SIZE - (payl_start - (void *)uframe_p)); + + if (wire_size != NULL) { + (*wire_size) += desc->len; + } + } + + xsk_ring_cons__release(&socket->rx, available); + *count = available; + + return KNOT_EOK; +} + +static uint8_t *msg_uframe_ptr(const knot_xdp_msg_t *msg) +{ + return NULL + ((msg->payload.iov_base - NULL) & ~(FRAME_SIZE - 1)); +} + +_public_ +void knot_xdp_recv_finish(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count) +{ + if (socket == NULL || msgs == NULL) { + return; + } + + struct kxsk_umem *const umem = socket->umem; + struct xsk_ring_prod *const fq = &umem->fq; + + uint32_t idx = 0; + const uint32_t reserved = xsk_ring_prod__reserve(fq, count, &idx); + assert(reserved == count); + + for (uint32_t i = 0; i < reserved; ++i) { + uint8_t *uframe_p = msg_uframe_ptr(&msgs[i]); + uint64_t offset = uframe_p - umem->frames->bytes; + *xsk_ring_prod__fill_addr(fq, idx++) = offset; + } + + xsk_ring_prod__submit(fq, reserved); +} + +_public_ +void knot_xdp_info(const knot_xdp_socket_t *socket, FILE *file) +{ + if (socket == NULL || file == NULL) { + return; + } + + // The number of busy frames + #define RING_BUSY(ring) \ + ((*(ring)->producer - *(ring)->consumer) & (ring)->mask) + + #define RING_PRINFO(name, ring) \ + fprintf(file, "Ring %s: size %4d, busy %4d (prod %4d, cons %4d)\n", \ + name, (unsigned)(ring)->size, \ + (unsigned)RING_BUSY((ring)), \ + (unsigned)*(ring)->producer, (unsigned)*(ring)->consumer) + + const int rx_busyf = RING_BUSY(&socket->umem->fq) + RING_BUSY(&socket->rx); + fprintf(file, "\nLOST RX frames: %4d", (int)(FRAME_COUNT_RX - rx_busyf)); + + const int tx_busyf = RING_BUSY(&socket->umem->cq) + RING_BUSY(&socket->tx); + const int tx_freef = socket->umem->tx_free_count; + fprintf(file, "\nLOST TX frames: %4d\n", (int)(FRAME_COUNT_TX - tx_busyf - tx_freef)); + + RING_PRINFO("FQ", &socket->umem->fq); + RING_PRINFO("RX", &socket->rx); + RING_PRINFO("TX", &socket->tx); + RING_PRINFO("CQ", &socket->umem->cq); + fprintf(file, "TX free frames: %4d\n", tx_freef); +} diff --git a/src/libknot/xdp/xdp.h b/src/libknot/xdp/xdp.h new file mode 100644 index 0000000..a7606fe --- /dev/null +++ b/src/libknot/xdp/xdp.h @@ -0,0 +1,189 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP IO interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <netinet/in.h> + +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/msg.h" + +/*! + * \brief Styles of loading BPF program. + * + * \note In *all* the cases loading can only succeed if at the end + * a compatible BPF program is loaded on the interface. + */ +typedef enum { + KNOT_XDP_LOAD_BPF_NEVER, /*!< Do not load; error out if not loaded already. */ + KNOT_XDP_LOAD_BPF_ALWAYS, /*!< Always load a program (overwrite it). */ + KNOT_XDP_LOAD_BPF_ALWAYS_UNLOAD, /*!< KNOT_XDP_LOAD_BPF_ALWAYS + unload previous. */ + KNOT_XDP_LOAD_BPF_MAYBE, /*!< Try with present program or load if none. */ + /* Implementation caveat: when re-using program in _MAYBE case, we get a message: + * libbpf: Kernel error message: XDP program already attached */ +} knot_xdp_load_bpf_t; + +/*! \brief Context structure for one XDP socket. */ +typedef struct knot_xdp_socket knot_xdp_socket_t; + +/*! + * \brief Initialize XDP socket. + * + * \param socket XDP socket. + * \param if_name Name of the net iface (e.g. eth0). + * \param if_queue Network card queue to be used (normally 1 socket per each queue). + * \param flags XDP filter configuration flags. + * \param udp_port UDP and/or TCP port to listen on if enabled via \a opts. + * \param quic_port QUIC/UDP port to listen on if enabled via \a opts. + * \param load_bpf Insert BPF program into packet processing. + * \param xdp_config Optional XDP socket configuration. + * + * \return KNOT_E* or -errno + */ +int knot_xdp_init(knot_xdp_socket_t **socket, const char *if_name, int if_queue, + knot_xdp_filter_flag_t flags, uint16_t udp_port, uint16_t quic_port, + knot_xdp_load_bpf_t load_bpf, const void *xdp_config); + +/*! + * \brief De-init XDP socket. + * + * \param socket XDP socket. + */ +void knot_xdp_deinit(knot_xdp_socket_t *socket); + +/*! + * \brief Return a file descriptor to be polled on for incoming packets. + * + * \param socket XDP socket. + * + * \return KNOT_E* + */ +int knot_xdp_socket_fd(knot_xdp_socket_t *socket); + +/*! + * \brief Collect completed TX buffers, so they can be used by knot_xdp_send_alloc(). + * + * \param socket XDP socket. + */ +void knot_xdp_send_prepare(knot_xdp_socket_t *socket); + +/*! + * \brief Allocate one buffer for an outgoing packet. + * + * \param socket XDP socket. + * \param flags Flags for new message. + * \param out Out: the allocated packet buffer. + * + * \return KNOT_E* + */ +int knot_xdp_send_alloc(knot_xdp_socket_t *socket, knot_xdp_msg_flag_t flags, + knot_xdp_msg_t *out); + +/*! + * \brief Allocate one buffer for a reply packet. + * + * \param socket XDP socket. + * \param query The packet to be replied to. + * \param out Out: the allocated packet buffer. + * + * \return KNOT_E* + */ +int knot_xdp_reply_alloc(knot_xdp_socket_t *socket, const knot_xdp_msg_t *query, + knot_xdp_msg_t *out); + +/*! + * \brief Send multiple packets thru XDP. + * + * \note The packets all must have been allocated by knot_xdp_send_alloc()! + * \note Do not free the packet payloads afterwards. + * \note Packets with zero length will be skipped. + * + * \param socket XDP socket. + * \param msgs Packets to be sent. + * \param count Number of packets. + * \param sent Out: number of packet successfully sent. + * + * \return KNOT_E* + */ +int knot_xdp_send(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count, uint32_t *sent); + +/*! + * \brief Cleanup messages that have not been knot_xdp_send(). + * + * ...possibly due to some error. + * + * \param socket XDP socket. + * \param msgs Messages to be freed. + * \param count Number of messages. + */ +void knot_xdp_send_free(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count); + +/*! + * \brief Syscall to kernel to wake up the network card driver after knot_xdp_send(). + * + * \param socket XDP socket. + * + * \return KNOT_E* or -errno + */ +int knot_xdp_send_finish(knot_xdp_socket_t *socket); + +/*! + * \brief Receive multiple packets thru XDP. + * + * \param socket XDP socket. + * \param msgs Out: buffers to be filled in with incoming packets. + * \param max_count Limit for number of packets received at once. + * \param count Out: real number of received packets. + * \param wire_size Out: (optional) total wire size of received packets. + * + * \return KNOT_E* + */ +int knot_xdp_recv(knot_xdp_socket_t *socket, knot_xdp_msg_t msgs[], + uint32_t max_count, uint32_t *count, size_t *wire_size); + +/*! + * \brief Free buffers with received packets. + * + * \param socket XDP socket. + * \param msgs Buffers with received packets. + * \param count Number of received packets to free. + */ +void knot_xdp_recv_finish(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count); + +/*! + * \brief Print some info about the XDP socket. + * + * \param socket XDP socket. + * \param file Output file. + */ +void knot_xdp_info(const knot_xdp_socket_t *socket, FILE *file); + +/*! @} */ |