diff options
Diffstat (limited to 'src/libknot/xdp')
| -rw-r--r-- | src/libknot/xdp/Makefile.am | 20 | ||||
| -rw-r--r-- | src/libknot/xdp/Makefile.in | 544 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-consts.h | 57 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel-obj.c | 941 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel-obj.h | 2 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-kernel.c | 293 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-user.c | 319 | ||||
| -rw-r--r-- | src/libknot/xdp/bpf-user.h | 140 | ||||
| -rw-r--r-- | src/libknot/xdp/eth.c | 312 | ||||
| -rw-r--r-- | src/libknot/xdp/eth.h | 111 | ||||
| -rw-r--r-- | src/libknot/xdp/msg.h | 63 | ||||
| -rw-r--r-- | src/libknot/xdp/msg_init.h | 74 | ||||
| -rw-r--r-- | src/libknot/xdp/protocols.h | 457 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp.c | 749 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp.h | 226 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp_iobuf.c | 302 | ||||
| -rw-r--r-- | src/libknot/xdp/tcp_iobuf.h | 129 | ||||
| -rw-r--r-- | src/libknot/xdp/xdp.c | 585 | ||||
| -rw-r--r-- | src/libknot/xdp/xdp.h | 199 |
19 files changed, 5523 insertions, 0 deletions
diff --git a/src/libknot/xdp/Makefile.am b/src/libknot/xdp/Makefile.am new file mode 100644 index 0000000..cb58438 --- /dev/null +++ b/src/libknot/xdp/Makefile.am @@ -0,0 +1,20 @@ +# Useful commands: +# make filter +# ip link show $eth +# sudo ip link set dev $eth xdp off +# sudo ip link set dev $eth xdp obj ./bpf-kernel.o +# +# Built using LLVM/CLANG 14. +# +# When updating check using `llvm-objdump -h bpf-kernel.o` if .BTF and .BTF.ext +# sections are present. + +EXTRA_DIST = bpf-kernel-obj.c bpf-kernel.c + +.PHONY: filter + +filter: + rm -f bpf-kernel.o bpf-kernel-obj.c + clang -target bpf -Wall -O2 -g -DNDEBUG -c -o bpf-kernel.o -I/usr/include/x86_64-linux-gnu -include ../../config.h bpf-kernel.c + llvm-strip -S bpf-kernel.o + xxd -i bpf-kernel.o > bpf-kernel-obj.c diff --git a/src/libknot/xdp/Makefile.in b/src/libknot/xdp/Makefile.in new file mode 100644 index 0000000..8aa77ce --- /dev/null +++ b/src/libknot/xdp/Makefile.in @@ -0,0 +1,544 @@ +# Makefile.in generated by automake 1.16.5 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2021 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +# Useful commands: +# make filter +# ip link show $eth +# sudo ip link set dev $eth xdp off +# sudo ip link set dev $eth xdp obj ./bpf-kernel.o +# +# Built using LLVM/CLANG 14. +# +# When updating check using `llvm-objdump -h bpf-kernel.o` if .BTF and .BTF.ext +# sections are present. +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/libknot/xdp +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_check_link_flag.m4 \ + $(top_srcdir)/m4/code-coverage.m4 \ + $(top_srcdir)/m4/knot-lib-version.m4 \ + $(top_srcdir)/m4/knot-module.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/sanitizer.m4 $(top_srcdir)/m4/visibility.m4 \ + $(top_srcdir)/m4/knot-version.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/src/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +SOURCES = +DIST_SOURCES = +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +am__DIST_COMMON = $(srcdir)/Makefile.in +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CFLAG_VISIBILITY = @CFLAG_VISIBILITY@ +CODE_COVERAGE_ENABLED = @CODE_COVERAGE_ENABLED@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CSCOPE = @CSCOPE@ +CTAGS = @CTAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DNSTAP_CFLAGS = @DNSTAP_CFLAGS@ +DNSTAP_LIBS = @DNSTAP_LIBS@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +ETAGS = @ETAGS@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FILECMD = @FILECMD@ +GENHTML = @GENHTML@ +GREP = @GREP@ +HAVE_VISIBILITY = @HAVE_VISIBILITY@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KNOT_VERSION_MAJOR = @KNOT_VERSION_MAJOR@ +KNOT_VERSION_MINOR = @KNOT_VERSION_MINOR@ +KNOT_VERSION_PATCH = @KNOT_VERSION_PATCH@ +LCOV = @LCOV@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LDFLAG_EXCLUDE_LIBS = @LDFLAG_EXCLUDE_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +LT_NO_UNDEFINED = @LT_NO_UNDEFINED@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PDFLATEX = @PDFLATEX@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +PROTOC_C = @PROTOC_C@ +RANLIB = @RANLIB@ +RELEASE_DATE = @RELEASE_DATE@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SPHINXBUILD = @SPHINXBUILD@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +cap_ng_CFLAGS = @cap_ng_CFLAGS@ +cap_ng_LIBS = @cap_ng_LIBS@ +conf_mapsize = @conf_mapsize@ +config_dir = @config_dir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dlopen_LIBS = @dlopen_LIBS@ +docdir = @docdir@ +dvidir = @dvidir@ +embedded_libngtcp2_CFLAGS = @embedded_libngtcp2_CFLAGS@ +embedded_libngtcp2_LIBS = @embedded_libngtcp2_LIBS@ +exec_prefix = @exec_prefix@ +fuzzer_CFLAGS = @fuzzer_CFLAGS@ +fuzzer_LDFLAGS = @fuzzer_LDFLAGS@ +gnutls_CFLAGS = @gnutls_CFLAGS@ +gnutls_LIBS = @gnutls_LIBS@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libbpf_CFLAGS = @libbpf_CFLAGS@ +libbpf_LIBS = @libbpf_LIBS@ +libdir = @libdir@ +libdnssec_SONAME = @libdnssec_SONAME@ +libdnssec_SOVERSION = @libdnssec_SOVERSION@ +libdnssec_VERSION_INFO = @libdnssec_VERSION_INFO@ +libedit_CFLAGS = @libedit_CFLAGS@ +libedit_LIBS = @libedit_LIBS@ +libexecdir = @libexecdir@ +libfstrm_CFLAGS = @libfstrm_CFLAGS@ +libfstrm_LIBS = @libfstrm_LIBS@ +libidn2_CFLAGS = @libidn2_CFLAGS@ +libidn2_LIBS = @libidn2_LIBS@ +libidn_CFLAGS = @libidn_CFLAGS@ +libidn_LIBS = @libidn_LIBS@ +libknot_SONAME = @libknot_SONAME@ +libknot_SOVERSION = @libknot_SOVERSION@ +libknot_VERSION_INFO = @libknot_VERSION_INFO@ +libkqueue_CFLAGS = @libkqueue_CFLAGS@ +libkqueue_LIBS = @libkqueue_LIBS@ +libmaxminddb_CFLAGS = @libmaxminddb_CFLAGS@ +libmaxminddb_LIBS = @libmaxminddb_LIBS@ +libmnl_CFLAGS = @libmnl_CFLAGS@ +libmnl_LIBS = @libmnl_LIBS@ +libnghttp2_CFLAGS = @libnghttp2_CFLAGS@ +libnghttp2_LIBS = @libnghttp2_LIBS@ +libngtcp2_CFLAGS = @libngtcp2_CFLAGS@ +libngtcp2_LIBS = @libngtcp2_LIBS@ +libprotobuf_c_CFLAGS = @libprotobuf_c_CFLAGS@ +libprotobuf_c_LIBS = @libprotobuf_c_LIBS@ +liburcu_CFLAGS = @liburcu_CFLAGS@ +liburcu_LIBS = @liburcu_LIBS@ +liburcu_PKGCONFIG = @liburcu_PKGCONFIG@ +libxdp_CFLAGS = @libxdp_CFLAGS@ +libxdp_LIBS = @libxdp_LIBS@ +libzscanner_SONAME = @libzscanner_SONAME@ +libzscanner_SOVERSION = @libzscanner_SOVERSION@ +libzscanner_VERSION_INFO = @libzscanner_VERSION_INFO@ +lmdb_CFLAGS = @lmdb_CFLAGS@ +lmdb_LIBS = @lmdb_LIBS@ +localedir = @localedir@ +localstatedir = @localstatedir@ +malloc_LIBS = @malloc_LIBS@ +mandir = @mandir@ +math_LIBS = @math_LIBS@ +mkdir_p = @mkdir_p@ +module_dir = @module_dir@ +module_instdir = @module_instdir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgconfigdir = @pkgconfigdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pthread_LIBS = @pthread_LIBS@ +run_dir = @run_dir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +storage_dir = @storage_dir@ +sysconfdir = @sysconfdir@ +systemd_CFLAGS = @systemd_CFLAGS@ +systemd_LIBS = @systemd_LIBS@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +EXTRA_DIST = bpf-kernel-obj.c bpf-kernel.c +all: all-am + +.SUFFIXES: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/libknot/xdp/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/libknot/xdp/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +tags TAGS: + +ctags CTAGS: + +cscope cscopelist: + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool mostlyclean-am + +distclean: distclean-am + -rm -f Makefile +distclean-am: clean-am distclean-generic + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-generic mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: all all-am check check-am clean clean-generic clean-libtool \ + cscopelist-am ctags-am distclean distclean-generic \ + distclean-libtool distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +.PHONY: filter + +filter: + rm -f bpf-kernel.o bpf-kernel-obj.c + clang -target bpf -Wall -O2 -g -DNDEBUG -c -o bpf-kernel.o -I/usr/include/x86_64-linux-gnu -include ../../config.h bpf-kernel.c + llvm-strip -S bpf-kernel.o + xxd -i bpf-kernel.o > bpf-kernel-obj.c + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/libknot/xdp/bpf-consts.h b/src/libknot/xdp/bpf-consts.h new file mode 100644 index 0000000..3b92cbb --- /dev/null +++ b/src/libknot/xdp/bpf-consts.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP filter configuration constants. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <linux/types.h> + +#define KNOT_XDP_PKT_ALIGNMENT 2 /*!< Fix for misaligned access to packet structures. */ + +/*! \brief XDP filter configuration flags. */ +typedef enum { + KNOT_XDP_FILTER_ON = 1 << 0, /*!< Filter enabled. */ + KNOT_XDP_FILTER_UDP = 1 << 1, /*!< Apply filter to UDP. */ + KNOT_XDP_FILTER_TCP = 1 << 2, /*!< Apply filter to TCP. */ + KNOT_XDP_FILTER_QUIC = 1 << 3, /*!< Apply filter to QUIC/UDP. */ + KNOT_XDP_FILTER_PASS = 1 << 4, /*!< Pass incoming messages to ports >= port value. */ + KNOT_XDP_FILTER_DROP = 1 << 5, /*!< Drop incoming messages to ports >= port value. */ + KNOT_XDP_FILTER_ROUTE = 1 << 6, /*!< Consider routing information from kernel. */ +} knot_xdp_filter_flag_t; + +/*! \brief XDP map item for the filter configuration. */ +typedef struct knot_xdp_opts knot_xdp_opts_t; +struct knot_xdp_opts { + __u16 flags; /*!< XDP filter flags \a knot_xdp_filter_flag_t. */ + __u16 udp_port; /*!< UDP/TCP port to listen on. */ + __u16 quic_port; /*!< QUIC/UDP port to listen on. */ +} __attribute__((packed)); + +/*! \brief Additional information from the filter. */ +typedef struct knot_xdp_info knot_xdp_info_t; +struct knot_xdp_info { + __u16 out_if_index; /*!< Index of the output interface (if routing enabled). */ +}; + +/*! @} */ diff --git a/src/libknot/xdp/bpf-kernel-obj.c b/src/libknot/xdp/bpf-kernel-obj.c new file mode 100644 index 0000000..9f1ee71 --- /dev/null +++ b/src/libknot/xdp/bpf-kernel-obj.c @@ -0,0 +1,941 @@ +unsigned char bpf_kernel_o[] = { + 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x0d, 0x00, 0x01, 0x00, 0xbf, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x61, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x1a, 0xfc, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x02, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, 0x18, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x71, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x09, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x29, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x15, 0x02, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xa0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0x98, 0xff, 0x00, 0x00, 0x00, 0x00, 0x71, 0x18, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x11, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x1a, 0xa8, 0xff, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, + 0x36, 0x00, 0x00, 0x00, 0x61, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x61, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x72, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x3a, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x62, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x21, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x74, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x73, 0x0d, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x4f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0b, 0x00, + 0x81, 0x00, 0x00, 0x00, 0xbf, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x01, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x21, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xb0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x74, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x73, 0x11, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x4f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x15, 0x03, 0x1e, 0x00, 0x86, 0xdd, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x55, 0x03, 0xdc, 0x00, + 0x08, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x34, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x04, 0xd3, 0x00, + 0x40, 0x00, 0x00, 0x00, 0xbf, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x15, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6d, 0x45, 0xcd, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x15, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x05, 0x00, 0x00, 0xbf, 0xff, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x4a, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x05, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x3c, 0x00, 0x00, 0x00, 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x10, 0x09, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x05, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x2d, 0x25, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x03, 0xb8, 0x00, + 0x60, 0x00, 0x00, 0x00, 0xbf, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x14, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x04, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x07, 0x04, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x34, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7b, 0x3a, 0x90, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x10, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x08, 0x00, + 0x2c, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xbf, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x05, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x2d, 0x25, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x71, 0x10, 0x28, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x7b, 0x3a, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, 0x67, 0x08, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xa8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x4f, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x17, 0x00, 0x11, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x55, 0x03, 0x9d, 0x00, + 0x06, 0x00, 0x00, 0x00, 0xbf, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x52, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x02, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xbf, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x1d, 0x32, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x8c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x08, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x2d, 0x28, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x28, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x53, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6d, 0x23, 0x7f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x52, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x02, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x83, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x1d, 0x32, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x08, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x3d, 0x82, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x15, 0x03, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0xa3, 0xa0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x79, 0xa5, 0x98, 0xff, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x53, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x03, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x1d, 0x32, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xbf, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x03, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x15, 0x03, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x57, 0x05, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x2d, 0x25, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x55, 0x02, 0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, + 0xff, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x55, 0x04, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x09, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x15, 0x09, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xf0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xe8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xe0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xd0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc8, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x63, 0x2a, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa2, 0x90, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x02, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x73, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x2a, 0xc8, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x11, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x1a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x73, 0x2a, 0xb8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x61, 0x13, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x32, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xc8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x12, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x61, 0x13, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x2a, 0xd0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x61, 0x13, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0x2a, 0xe0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x61, 0x12, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x11, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x4f, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0x1a, 0xd8, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, + 0xb8, 0xff, 0xff, 0xff, 0xbf, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x24, 0x00, 0x07, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x21, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x69, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xec, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x71, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xee, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x19, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x71, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa2, 0xf0, 0xff, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x21, 0x16, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa1, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0xa1, 0xc0, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xa2, 0xb0, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0xa1, 0xf6, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x73, 0x17, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x01, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x73, 0x17, 0x0b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0xa1, 0xf4, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x17, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x73, 0x17, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xa1, 0xf2, 0xff, 0x00, 0x00, 0x00, 0x00, 0x73, 0x17, 0x06, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x73, 0x17, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x62, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0xfd, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x00, 0xfb, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x50, 0x4c, 0x00, + 0x9f, 0xeb, 0x01, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7c, 0x02, 0x00, 0x00, 0x7c, 0x02, 0x00, 0x00, 0xce, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x04, 0x20, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0e, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x04, 0x20, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x0f, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x12, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x04, + 0x18, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x14, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x0d, 0x02, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x0c, + 0x15, 0x00, 0x00, 0x00, 0xb2, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb7, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0e, 0x18, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x0a, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xc6, 0x0a, 0x00, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x69, 0x6e, 0x74, 0x00, 0x5f, 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, + 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5f, + 0x00, 0x74, 0x79, 0x70, 0x65, 0x00, 0x6d, 0x61, 0x78, 0x5f, 0x65, 0x6e, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x00, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x69, + 0x7a, 0x65, 0x00, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x00, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, + 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, 0x64, 0x70, 0x5f, + 0x6d, 0x64, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x65, 0x6e, 0x64, 0x00, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x00, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x69, + 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x72, 0x78, 0x5f, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x65, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x5f, 0x69, 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x00, 0x5f, 0x5f, 0x75, 0x33, 0x32, 0x00, 0x75, 0x6e, 0x73, 0x69, 0x67, + 0x6e, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x74, 0x00, 0x63, 0x74, 0x78, 0x00, + 0x78, 0x64, 0x70, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x5f, 0x64, 0x6e, 0x73, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x00, 0x78, 0x64, + 0x70, 0x00, 0x2f, 0x68, 0x6f, 0x6d, 0x65, 0x2f, 0x64, 0x73, 0x61, 0x6c, + 0x7a, 0x6d, 0x61, 0x6e, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x32, 0x2f, 0x73, + 0x72, 0x63, 0x2f, 0x6c, 0x69, 0x62, 0x6b, 0x6e, 0x6f, 0x74, 0x2f, 0x78, + 0x64, 0x70, 0x2f, 0x62, 0x70, 0x66, 0x2d, 0x6b, 0x65, 0x72, 0x6e, 0x65, + 0x6c, 0x2e, 0x63, 0x00, 0x69, 0x6e, 0x74, 0x20, 0x78, 0x64, 0x70, 0x5f, + 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x6e, 0x73, + 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x20, 0x78, 0x64, 0x70, 0x5f, 0x6d, 0x64, 0x20, 0x2a, 0x63, 0x74, 0x78, + 0x29, 0x00, 0x09, 0x5f, 0x5f, 0x75, 0x33, 0x32, 0x20, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x20, 0x3d, 0x20, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x72, 0x78, + 0x5f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x3b, 0x00, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x6b, 0x6e, + 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x6f, 0x70, 0x74, 0x73, 0x20, + 0x2a, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x70, 0x74, 0x72, 0x20, 0x3d, 0x20, + 0x62, 0x70, 0x66, 0x5f, 0x6d, 0x61, 0x70, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x28, 0x26, 0x6f, 0x70, 0x74, + 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x2c, 0x20, 0x26, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x29, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x21, 0x6f, 0x70, + 0x74, 0x73, 0x5f, 0x70, 0x74, 0x72, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x21, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, + 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, + 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x4f, 0x4e, + 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x29, + 0x62, 0x70, 0x66, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x61, 0x64, 0x6a, 0x75, + 0x73, 0x74, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x28, 0x63, 0x74, 0x78, 0x2c, + 0x20, 0x2d, 0x20, 0x28, 0x69, 0x6e, 0x74, 0x29, 0x73, 0x69, 0x7a, 0x65, + 0x6f, 0x66, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x6b, 0x6e, + 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x29, + 0x00, 0x09, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x64, 0x61, 0x74, 0x61, + 0x20, 0x3d, 0x20, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, + 0x6c, 0x6f, 0x6e, 0x67, 0x29, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, + 0x74, 0x61, 0x3b, 0x00, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, + 0x6b, 0x6e, 0x6f, 0x74, 0x5f, 0x78, 0x64, 0x70, 0x5f, 0x69, 0x6e, 0x66, + 0x6f, 0x20, 0x2a, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, 0x6c, 0x6f, 0x6e, 0x67, 0x29, + 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x2a, 0x29, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x6d, 0x65, 0x74, 0x61, + 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x2a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x3d, 0x20, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x28, 0x6c, 0x6f, 0x6e, + 0x67, 0x29, 0x63, 0x74, 0x78, 0x2d, 0x3e, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x65, 0x6e, 0x64, 0x3b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, + 0x72, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, + 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, 0x72, 0x29, 0x20, 0x3e, 0x20, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x69, 0x66, 0x20, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x68, 0x64, 0x72, 0x2d, + 0x3e, 0x68, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x20, 0x3d, 0x3d, 0x20, + 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x68, + 0x74, 0x6f, 0x6e, 0x73, 0x28, 0x45, 0x54, 0x48, 0x5f, 0x50, 0x5f, 0x38, + 0x30, 0x32, 0x31, 0x51, 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, 0x73, 0x69, + 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x5f, 0x5f, 0x75, 0x31, 0x36, 0x29, 0x20, + 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x65, 0x74, 0x68, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x65, + 0x74, 0x61, 0x20, 0x3d, 0x3d, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x09, 0x5f, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x5f, 0x6d, + 0x65, 0x6d, 0x63, 0x70, 0x79, 0x28, 0x26, 0x65, 0x74, 0x68, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x5f, 0x5f, 0x75, 0x31, 0x36, + 0x29, 0x2c, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x65, 0x74, + 0x68, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x29, 0x29, 0x3b, 0x00, 0x09, 0x73, + 0x77, 0x69, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, + 0x28, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x69, 0x70, 0x34, + 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x69, + 0x70, 0x34, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, + 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, + 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x21, 0x3d, 0x20, 0x34, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, + 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, + 0x2d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3c, 0x20, 0x5f, 0x5f, 0x62, + 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x69, 0x70, 0x34, + 0x2d, 0x3e, 0x74, 0x6f, 0x74, 0x5f, 0x6c, 0x65, 0x6e, 0x29, 0x29, 0x20, + 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, 0x70, 0x34, 0x2d, + 0x3e, 0x66, 0x72, 0x61, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x20, 0x21, 0x3d, + 0x20, 0x30, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, 0x6c, 0x34, 0x5f, 0x68, + 0x64, 0x72, 0x20, 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x2b, 0x20, + 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x69, 0x68, 0x6c, 0x20, 0x2a, 0x20, 0x34, + 0x3b, 0x00, 0x09, 0x09, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x20, 0x3d, 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x3b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x2a, 0x29, 0x69, 0x70, 0x36, 0x20, + 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x69, 0x70, + 0x36, 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, + 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, + 0x70, 0x36, 0x2d, 0x3e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, + 0x21, 0x3d, 0x20, 0x36, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x2d, + 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3c, 0x20, 0x5f, 0x5f, 0x62, 0x70, + 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x69, 0x70, 0x36, 0x2d, + 0x3e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6c, 0x65, 0x6e, + 0x29, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, + 0x69, 0x70, 0x36, 0x29, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x70, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x36, + 0x2d, 0x3e, 0x6e, 0x65, 0x78, 0x74, 0x68, 0x64, 0x72, 0x3b, 0x00, 0x09, + 0x09, 0x69, 0x66, 0x20, 0x28, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x20, 0x3d, 0x3d, 0x20, 0x49, 0x50, 0x50, 0x52, 0x4f, 0x54, 0x4f, + 0x5f, 0x46, 0x52, 0x41, 0x47, 0x4d, 0x45, 0x4e, 0x54, 0x29, 0x20, 0x7b, + 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x2a, 0x29, 0x66, 0x72, 0x61, 0x67, 0x20, 0x2b, 0x20, 0x73, + 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x66, 0x72, 0x61, 0x67, 0x29, + 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, + 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x66, 0x72, 0x61, 0x67, 0x2d, 0x3e, 0x6e, + 0x65, 0x78, 0x74, 0x68, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x73, 0x77, 0x69, + 0x74, 0x63, 0x68, 0x20, 0x28, 0x69, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6c, + 0x34, 0x5f, 0x68, 0x64, 0x72, 0x20, 0x2b, 0x20, 0x73, 0x69, 0x7a, 0x65, + 0x6f, 0x66, 0x28, 0x2a, 0x74, 0x63, 0x70, 0x29, 0x20, 0x3e, 0x20, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x29, 0x20, 0x7b, 0x00, 0x09, + 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, + 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, + 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x54, + 0x43, 0x50, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, 0x70, 0x6f, 0x72, + 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, 0x3d, 0x20, 0x5f, 0x5f, 0x62, + 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x74, 0x63, 0x70, + 0x2d, 0x3e, 0x64, 0x65, 0x73, 0x74, 0x29, 0x3b, 0x00, 0x09, 0x09, 0x20, + 0x20, 0x20, 0x20, 0x28, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x65, 0x73, + 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x75, 0x64, + 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x09, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x28, 0x4b, 0x4e, 0x4f, + 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, + 0x5f, 0x50, 0x41, 0x53, 0x53, 0x20, 0x7c, 0x20, 0x4b, 0x4e, 0x4f, 0x54, + 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, + 0x44, 0x52, 0x4f, 0x50, 0x29, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, + 0x69, 0x66, 0x20, 0x28, 0x6c, 0x34, 0x5f, 0x68, 0x64, 0x72, 0x20, 0x2b, + 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x2a, 0x75, 0x64, 0x70, + 0x29, 0x20, 0x3e, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, + 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x65, 0x6e, 0x64, 0x20, 0x2d, 0x20, 0x28, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x2a, 0x29, 0x75, 0x64, 0x70, 0x20, 0x3c, 0x20, 0x5f, + 0x5f, 0x62, 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, 0x73, 0x28, 0x75, + 0x64, 0x70, 0x2d, 0x3e, 0x6c, 0x65, 0x6e, 0x29, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x09, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, + 0x3d, 0x20, 0x5f, 0x5f, 0x62, 0x70, 0x66, 0x5f, 0x6e, 0x74, 0x6f, 0x68, + 0x73, 0x28, 0x75, 0x64, 0x70, 0x2d, 0x3e, 0x64, 0x65, 0x73, 0x74, 0x29, + 0x3b, 0x00, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x28, 0x6f, 0x70, 0x74, + 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, + 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, + 0x52, 0x5f, 0x55, 0x44, 0x50, 0x29, 0x20, 0x26, 0x26, 0x00, 0x09, 0x09, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, + 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, + 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, + 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x51, 0x55, 0x49, 0x43, 0x29, 0x20, 0x26, + 0x26, 0x00, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, 0x28, 0x70, 0x6f, 0x72, + 0x74, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x6f, 0x70, + 0x74, 0x73, 0x2e, 0x71, 0x75, 0x69, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x6f, 0x70, 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, + 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, + 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, 0x45, 0x52, 0x5f, 0x44, 0x52, 0x4f, + 0x50, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6f, 0x70, + 0x74, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x20, 0x26, 0x20, 0x4b, + 0x4e, 0x4f, 0x54, 0x5f, 0x58, 0x44, 0x50, 0x5f, 0x46, 0x49, 0x4c, 0x54, + 0x45, 0x52, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x29, 0x20, 0x7b, 0x00, + 0x09, 0x09, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x20, 0x62, 0x70, 0x66, + 0x5f, 0x66, 0x69, 0x62, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x20, + 0x66, 0x69, 0x62, 0x20, 0x3d, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x69, 0x70, 0x76, 0x34, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, + 0x09, 0x66, 0x69, 0x62, 0x2e, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x20, + 0x20, 0x20, 0x3d, 0x20, 0x41, 0x46, 0x5f, 0x49, 0x4e, 0x45, 0x54, 0x3b, + 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, 0x62, 0x2e, 0x69, 0x70, 0x76, 0x34, + 0x5f, 0x73, 0x72, 0x63, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, + 0x64, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, + 0x62, 0x2e, 0x69, 0x70, 0x76, 0x34, 0x5f, 0x64, 0x73, 0x74, 0x20, 0x3d, + 0x20, 0x69, 0x70, 0x34, 0x2d, 0x3e, 0x73, 0x61, 0x64, 0x64, 0x72, 0x3b, + 0x00, 0x09, 0x09, 0x09, 0x66, 0x69, 0x62, 0x2e, 0x66, 0x61, 0x6d, 0x69, + 0x6c, 0x79, 0x20, 0x3d, 0x20, 0x41, 0x46, 0x5f, 0x49, 0x4e, 0x45, 0x54, + 0x36, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x2a, 0x69, 0x70, 0x76, 0x36, 0x5f, + 0x73, 0x72, 0x63, 0x20, 0x20, 0x3d, 0x20, 0x69, 0x70, 0x36, 0x2d, 0x3e, + 0x64, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, 0x09, 0x09, 0x2a, 0x69, + 0x70, 0x76, 0x36, 0x5f, 0x64, 0x73, 0x74, 0x20, 0x20, 0x3d, 0x20, 0x69, + 0x70, 0x36, 0x2d, 0x3e, 0x73, 0x61, 0x64, 0x64, 0x72, 0x3b, 0x00, 0x09, + 0x09, 0x69, 0x6e, 0x74, 0x20, 0x72, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x62, + 0x70, 0x66, 0x5f, 0x66, 0x69, 0x62, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, + 0x70, 0x28, 0x63, 0x74, 0x78, 0x2c, 0x20, 0x26, 0x66, 0x69, 0x62, 0x2c, + 0x20, 0x73, 0x69, 0x7a, 0x65, 0x6f, 0x66, 0x28, 0x66, 0x69, 0x62, 0x29, + 0x2c, 0x20, 0x42, 0x50, 0x46, 0x5f, 0x46, 0x49, 0x42, 0x5f, 0x4c, 0x4f, + 0x4f, 0x4b, 0x55, 0x50, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x29, + 0x3b, 0x00, 0x09, 0x09, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68, 0x20, 0x28, + 0x72, 0x65, 0x74, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, + 0x20, 0x28, 0x6d, 0x61, 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x30, 0x5d, 0x20, + 0x21, 0x3d, 0x20, 0x6d, 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x30, + 0x5d, 0x20, 0x7c, 0x7c, 0x00, 0x09, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, + 0x6d, 0x61, 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x31, 0x5d, 0x20, 0x21, 0x3d, + 0x20, 0x6d, 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x31, 0x5d, 0x20, + 0x7c, 0x7c, 0x00, 0x09, 0x09, 0x09, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, + 0x63, 0x5f, 0x69, 0x6e, 0x5b, 0x32, 0x5d, 0x20, 0x21, 0x3d, 0x20, 0x6d, + 0x61, 0x63, 0x5f, 0x6f, 0x75, 0x74, 0x5b, 0x32, 0x5d, 0x29, 0x20, 0x7b, + 0x00, 0x09, 0x09, 0x09, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x65, 0x74, 0x61, + 0x20, 0x21, 0x3d, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x00, 0x09, 0x09, 0x09, + 0x09, 0x6d, 0x65, 0x74, 0x61, 0x2d, 0x3e, 0x6f, 0x75, 0x74, 0x5f, 0x69, + 0x66, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x20, 0x3d, 0x20, 0x66, 0x69, + 0x62, 0x2e, 0x69, 0x66, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x3b, 0x00, 0x09, + 0x09, 0x09, 0x5f, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x5f, + 0x6d, 0x65, 0x6d, 0x63, 0x70, 0x79, 0x28, 0x65, 0x74, 0x68, 0x5f, 0x68, + 0x64, 0x72, 0x2d, 0x3e, 0x68, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2c, 0x20, 0x66, 0x69, 0x62, 0x2e, 0x64, 0x6d, 0x61, 0x63, 0x2c, 0x20, + 0x45, 0x54, 0x48, 0x5f, 0x41, 0x4c, 0x45, 0x4e, 0x29, 0x3b, 0x00, 0x09, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x62, 0x70, 0x66, 0x5f, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x6d, 0x61, 0x70, 0x28, + 0x26, 0x78, 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x2c, 0x20, 0x63, + 0x74, 0x78, 0x2d, 0x3e, 0x72, 0x78, 0x5f, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2c, 0x20, 0x30, 0x29, 0x3b, 0x00, + 0x7d, 0x00, 0x63, 0x68, 0x61, 0x72, 0x00, 0x5f, 0x6c, 0x69, 0x63, 0x65, + 0x6e, 0x73, 0x65, 0x00, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x00, 0x6c, 0x69, + 0x63, 0x65, 0x6e, 0x73, 0x65, 0x00, 0x00, 0x00, 0x9f, 0xeb, 0x01, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x2c, 0x07, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0xf0, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x2e, 0x01, 0x00, 0x00, 0x15, 0xfc, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, 0x08, 0xfc, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x52, 0x01, 0x00, 0x00, 0x23, 0x00, 0x01, 0x00, 0x50, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, 0x06, 0x04, 0x01, 0x00, + 0x58, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, + 0x08, 0x1c, 0x01, 0x00, 0x78, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xae, 0x01, 0x00, 0x00, 0x13, 0x1c, 0x01, 0x00, 0x90, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0x06, 0x1c, 0x01, 0x00, + 0x98, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xd9, 0x01, 0x00, 0x00, 0x08, 0x30, 0x01, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, 0x22, 0x3c, 0x01, 0x00, + 0xf8, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x44, 0x02, 0x00, 0x00, + 0x32, 0x44, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x80, 0x02, 0x00, 0x00, 0x13, 0x50, 0x01, 0x00, 0x18, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x06, 0x50, 0x01, 0x00, + 0x30, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xac, 0x02, 0x00, 0x00, + 0x2c, 0x40, 0x01, 0x00, 0x38, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xe1, 0x02, 0x00, 0x00, 0x16, 0x8c, 0x01, 0x00, 0x50, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xe1, 0x02, 0x00, 0x00, 0x06, 0x8c, 0x01, 0x00, + 0x58, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, + 0x0f, 0xa4, 0x01, 0x00, 0x78, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x17, 0x03, 0x00, 0x00, 0x06, 0xa4, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, 0x1c, 0xa8, 0x01, 0x00, + 0x98, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, + 0x07, 0xa8, 0x01, 0x00, 0xa8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x8d, 0x03, 0x00, 0x00, 0x0e, 0xb0, 0x01, 0x00, 0xb8, 0x01, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xa7, 0x03, 0x00, 0x00, 0x03, 0xbc, 0x01, 0x00, + 0xd8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, + 0x02, 0xe0, 0x01, 0x00, 0xf8, 0x01, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x03, 0x04, 0x00, 0x00, 0x13, 0xec, 0x01, 0x00, 0x10, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x07, 0xec, 0x01, 0x00, + 0x18, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x32, 0x04, 0x00, 0x00, + 0x0c, 0xf8, 0x01, 0x00, 0x20, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x32, 0x04, 0x00, 0x00, 0x14, 0xf8, 0x01, 0x00, 0x38, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x32, 0x04, 0x00, 0x00, 0x07, 0xf8, 0x01, 0x00, + 0x40, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x4d, 0x04, 0x00, 0x00, + 0x10, 0x10, 0x02, 0x00, 0x50, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x4d, 0x04, 0x00, 0x00, 0x19, 0x10, 0x02, 0x00, 0x68, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x4d, 0x04, 0x00, 0x00, 0x07, 0x10, 0x02, 0x00, + 0x70, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x82, 0x04, 0x00, 0x00, + 0x0c, 0x20, 0x02, 0x00, 0x78, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x82, 0x04, 0x00, 0x00, 0x1a, 0x20, 0x02, 0x00, 0xa8, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, 0x1c, 0x34, 0x02, 0x00, + 0xb8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, + 0x11, 0x34, 0x02, 0x00, 0xc8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xbe, 0x04, 0x00, 0x00, 0x13, 0x30, 0x02, 0x00, 0xd8, 0x02, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00, 0x13, 0x48, 0x02, 0x00, + 0xf0, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00, + 0x07, 0x48, 0x02, 0x00, 0xf8, 0x02, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x09, 0x05, 0x00, 0x00, 0x0c, 0x54, 0x02, 0x00, 0x00, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, 0x14, 0x54, 0x02, 0x00, + 0x10, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, + 0x07, 0x54, 0x02, 0x00, 0x18, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0x10, 0x6c, 0x02, 0x00, 0x28, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, 0x19, 0x6c, 0x02, 0x00, + 0x38, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, + 0x37, 0x6c, 0x02, 0x00, 0x48, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0x07, 0x6c, 0x02, 0x00, 0x68, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x6c, 0x05, 0x00, 0x00, 0x13, 0x7c, 0x02, 0x00, + 0x70, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x87, 0x05, 0x00, 0x00, + 0x07, 0x84, 0x02, 0x00, 0x80, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xad, 0x05, 0x00, 0x00, 0x15, 0x90, 0x02, 0x00, 0x90, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xad, 0x05, 0x00, 0x00, 0x08, 0x90, 0x02, 0x00, + 0xa0, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xdf, 0x05, 0x00, 0x00, + 0x15, 0x9c, 0x02, 0x00, 0xb8, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xfc, 0x05, 0x00, 0x00, 0x02, 0xe0, 0x02, 0x00, + 0xf0, 0x03, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x11, 0x06, 0x00, 0x00, + 0x0e, 0xf0, 0x02, 0x00, 0x08, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x11, 0x06, 0x00, 0x00, 0x07, 0xf0, 0x02, 0x00, 0x10, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, 0x13, 0x08, 0x03, 0x00, + 0x28, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, + 0x2a, 0x08, 0x03, 0x00, 0x30, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x67, 0x06, 0x00, 0x00, 0x0f, 0x00, 0x03, 0x00, 0x40, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, 0x12, 0x0c, 0x03, 0x00, + 0x50, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, + 0x23, 0x0c, 0x03, 0x00, 0x58, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb2, 0x06, 0x00, 0x00, 0x15, 0x10, 0x03, 0x00, 0x70, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x46, 0x10, 0x03, 0x00, + 0x98, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xfa, 0x06, 0x00, 0x00, + 0x0e, 0x30, 0x03, 0x00, 0xb0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xfa, 0x06, 0x00, 0x00, 0x07, 0x30, 0x03, 0x00, 0xb8, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, 0x10, 0x44, 0x03, 0x00, + 0xc0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, + 0x20, 0x44, 0x03, 0x00, 0xd8, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x24, 0x07, 0x00, 0x00, 0x07, 0x44, 0x03, 0x00, 0xe0, 0x04, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x5c, 0x07, 0x00, 0x00, 0x0f, 0x54, 0x03, 0x00, + 0xf0, 0x04, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x82, 0x07, 0x00, 0x00, + 0x13, 0x5c, 0x03, 0x00, 0x00, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x82, 0x07, 0x00, 0x00, 0x2a, 0x5c, 0x03, 0x00, 0x08, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, 0x12, 0x60, 0x03, 0x00, + 0x18, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, + 0x23, 0x60, 0x03, 0x00, 0x20, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb2, 0x06, 0x00, 0x00, 0x15, 0x64, 0x03, 0x00, 0x30, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x46, 0x64, 0x03, 0x00, + 0x48, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xae, 0x07, 0x00, 0x00, + 0x1a, 0x70, 0x03, 0x00, 0x60, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xae, 0x07, 0x00, 0x00, 0x32, 0x70, 0x03, 0x00, 0x70, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xe2, 0x07, 0x00, 0x00, + 0x12, 0x74, 0x03, 0x00, 0x98, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xe2, 0x07, 0x00, 0x00, 0x24, 0x74, 0x03, 0x00, 0xa0, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0x15, 0x78, 0x03, 0x00, + 0xb8, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, + 0x46, 0x78, 0x03, 0x00, 0xd8, 0x05, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x00, 0x00, 0x18, 0xac, 0x03, 0x00, 0xf0, 0x05, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x08, 0x08, 0x00, 0x00, 0x0d, 0xac, 0x03, 0x00, + 0x10, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x39, 0x08, 0x00, 0x00, + 0x11, 0xd0, 0x03, 0x00, 0x18, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x39, 0x08, 0x00, 0x00, 0x06, 0xd0, 0x03, 0x00, 0x28, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x64, 0x08, 0x00, 0x00, 0x19, 0xd4, 0x03, 0x00, + 0x78, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x84, 0x08, 0x00, 0x00, + 0x07, 0xe0, 0x03, 0x00, 0x90, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x92, 0x08, 0x00, 0x00, 0x11, 0xe4, 0x03, 0x00, 0x98, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xad, 0x08, 0x00, 0x00, 0x18, 0xe8, 0x03, 0x00, + 0xa0, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xad, 0x08, 0x00, 0x00, + 0x11, 0xe8, 0x03, 0x00, 0xa8, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xcb, 0x08, 0x00, 0x00, 0x18, 0xec, 0x03, 0x00, 0xb0, 0x06, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xcb, 0x08, 0x00, 0x00, 0x11, 0xec, 0x03, 0x00, + 0xc8, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xe9, 0x08, 0x00, 0x00, + 0x0f, 0xfc, 0x03, 0x00, 0xd0, 0x06, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x03, 0x09, 0x00, 0x00, 0x16, 0x00, 0x04, 0x00, 0x20, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x1f, 0x09, 0x00, 0x00, 0x16, 0x04, 0x04, 0x00, + 0x78, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x3b, 0x09, 0x00, 0x00, 0x0d, 0x18, 0x04, 0x00, 0xb8, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x86, 0x09, 0x00, 0x00, 0x03, 0x1c, 0x04, 0x00, + 0xe0, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, + 0x08, 0x28, 0x04, 0x00, 0xe8, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x97, 0x09, 0x00, 0x00, 0x15, 0x28, 0x04, 0x00, 0xf0, 0x07, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, 0x20, 0x28, 0x04, 0x00, + 0xf8, 0x07, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xb9, 0x09, 0x00, 0x00, + 0x08, 0x2c, 0x04, 0x00, 0x00, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xb9, 0x09, 0x00, 0x00, 0x15, 0x2c, 0x04, 0x00, 0x08, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb9, 0x09, 0x00, 0x00, 0x20, 0x2c, 0x04, 0x00, + 0x10, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xdb, 0x09, 0x00, 0x00, + 0x08, 0x30, 0x04, 0x00, 0x18, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0xdb, 0x09, 0x00, 0x00, 0x15, 0x30, 0x04, 0x00, 0x20, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x97, 0x09, 0x00, 0x00, 0x08, 0x28, 0x04, 0x00, + 0x28, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xfd, 0x09, 0x00, 0x00, + 0x08, 0x44, 0x04, 0x00, 0x38, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x11, 0x0a, 0x00, 0x00, 0x1e, 0x48, 0x04, 0x00, 0x40, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0x11, 0x0a, 0x00, 0x00, 0x18, 0x48, 0x04, 0x00, + 0x50, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x37, 0x0a, 0x00, 0x00, + 0x04, 0x58, 0x04, 0x00, 0xb0, 0x08, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, + 0x73, 0x0a, 0x00, 0x00, 0x09, 0x88, 0x04, 0x00, 0xd8, 0x08, 0x00, 0x00, + 0xce, 0x00, 0x00, 0x00, 0xb0, 0x0a, 0x00, 0x00, 0x01, 0x8c, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x28, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xa8, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xb8, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x98, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xd8, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x48, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xb0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xc0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x70, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xe0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x50, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4b, 0x00, 0x00, 0x00, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x11, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2b, 0x00, 0x00, 0x00, 0x11, 0x00, 0x05, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x00, 0x00, 0x00, 0x11, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0xb8, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x68, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, + 0x74, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x8c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x60, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x60, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa0, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xb0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd0, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xe0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x30, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x60, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x70, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x90, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xa0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xb0, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xd0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe0, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x30, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x50, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x60, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x90, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa0, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xb0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd0, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xe0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xf0, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x30, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x50, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x70, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe0, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf0, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x30, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x1a, 0x1b, 0x1c, 0x1d, 0x00, 0x2e, 0x74, 0x65, 0x78, 0x74, 0x00, 0x2e, + 0x72, 0x65, 0x6c, 0x2e, 0x42, 0x54, 0x46, 0x2e, 0x65, 0x78, 0x74, 0x00, + 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x00, 0x2e, 0x72, 0x65, 0x6c, 0x78, 0x64, + 0x70, 0x00, 0x6f, 0x70, 0x74, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x78, + 0x73, 0x6b, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x2e, 0x6c, 0x6c, 0x76, + 0x6d, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x69, 0x67, 0x00, 0x5f, 0x6c, + 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x00, 0x78, 0x64, 0x70, 0x5f, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x6e, 0x73, 0x5f, + 0x66, 0x75, 0x6e, 0x63, 0x00, 0x2e, 0x73, 0x74, 0x72, 0x74, 0x61, 0x62, + 0x00, 0x2e, 0x73, 0x79, 0x6d, 0x74, 0x61, 0x62, 0x00, 0x2e, 0x72, 0x65, + 0x6c, 0x2e, 0x42, 0x54, 0x46, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x39, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x35, 0x38, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x35, 0x37, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x31, 0x37, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x35, 0x36, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x34, 0x36, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x33, 0x36, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x31, 0x36, 0x00, 0x4c, 0x42, 0x42, + 0x30, 0x5f, 0x35, 0x35, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x35, + 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x00, 0x4c, 0x42, 0x42, 0x30, + 0x5f, 0x35, 0x34, 0x00, 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x32, 0x33, 0x00, + 0x4c, 0x42, 0x42, 0x30, 0x5f, 0x34, 0x30, 0x00, 0x4c, 0x42, 0x42, 0x30, + 0x5f, 0x33, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc4, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1a, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x09, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x0d, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x16, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x03, 0x4c, 0xff, 0x6f, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +unsigned int bpf_kernel_o_len = 11256; diff --git a/src/libknot/xdp/bpf-kernel-obj.h b/src/libknot/xdp/bpf-kernel-obj.h new file mode 100644 index 0000000..7c3d759 --- /dev/null +++ b/src/libknot/xdp/bpf-kernel-obj.h @@ -0,0 +1,2 @@ +extern unsigned char bpf_kernel_o[]; +extern unsigned int bpf_kernel_o_len; diff --git a/src/libknot/xdp/bpf-kernel.c b/src/libknot/xdp/bpf-kernel.c new file mode 100644 index 0000000..97192bc --- /dev/null +++ b/src/libknot/xdp/bpf-kernel.c @@ -0,0 +1,293 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <linux/bpf.h> +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <bpf/bpf_endian.h> +#include <bpf/bpf_helpers.h> + +#include "bpf-consts.h" + +/* Don't fragment flag. */ +#define IP_DF 0x4000 + +#define AF_INET 2 +#define AF_INET6 10 + +/* Define maximum reasonable number of NIC queues supported. */ +#define QUEUE_MAX 256 + +/* A map of configuration options. */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, QUEUE_MAX); + __uint(key_size, sizeof(__u32)); /* Must be 4 bytes. */ + __uint(value_size, sizeof(knot_xdp_opts_t)); +} opts_map SEC(".maps"); + +/* A map of AF_XDP sockets. */ +struct { + __uint(type, BPF_MAP_TYPE_XSKMAP); + __uint(max_entries, QUEUE_MAX); + __uint(key_size, sizeof(__u32)); /* Must be 4 bytes. */ + __uint(value_size, sizeof(int)); +} xsks_map SEC(".maps"); + +struct ipv6_frag_hdr { + unsigned char nexthdr; + unsigned char whatever[7]; +} __attribute__((packed)); + +SEC("xdp") +int xdp_redirect_dns_func(struct xdp_md *ctx) +{ + /* Get the queue options. */ + __u32 index = ctx->rx_queue_index; + struct knot_xdp_opts *opts_ptr = bpf_map_lookup_elem(&opts_map, &index); + if (!opts_ptr) { + return XDP_ABORTED; + } + knot_xdp_opts_t opts = *opts_ptr; + + /* Check if the filter is disabled. */ + if (!(opts.flags & KNOT_XDP_FILTER_ON)) { + return XDP_PASS; + } + + /* Try to reserve space in front of the packet for additional (VLAN) data. */ + (void)bpf_xdp_adjust_meta(ctx, - (int)sizeof(struct knot_xdp_info) + - KNOT_XDP_PKT_ALIGNMENT); + + void *data = (void *)(long)ctx->data; + const void *data_end = (void *)(long)ctx->data_end; + struct knot_xdp_info *meta = (void *)(long)ctx->data_meta; + + /* Check if the meta data pointer is usable (e.g. not `tap` interface). */ + if ((void *)meta + sizeof(*meta) > data) { + meta = 0; + } + + struct ethhdr *eth_hdr = data; + const void *ip_hdr; + const struct iphdr *ip4; + const struct ipv6hdr *ip6; + const void *l4_hdr; + __u8 ipv4; + __u8 ip_proto; + __u8 fragmented = 0; + __u16 eth_type; /* In big endian. */ + + /* Parse Ethernet header. */ + if ((void *)eth_hdr + sizeof(*eth_hdr) > data_end) { + return XDP_DROP; + } + data += sizeof(*eth_hdr); + + /* Parse possible VLAN (802.1Q) header. */ + if (eth_hdr->h_proto == __constant_htons(ETH_P_8021Q)) { + if (data + sizeof(__u16) + sizeof(eth_type) > data_end) { + return XDP_DROP; + } else if (meta == 0) { /* VLAN not supported. */ + return XDP_PASS; + } + __builtin_memcpy(ð_type, data + sizeof(__u16), sizeof(eth_type)); + data += sizeof(__u16) + sizeof(eth_type); + } else { + eth_type = eth_hdr->h_proto; + } + + ip_hdr = data; + + /* Parse IPv4 or IPv6 header. */ + switch (eth_type) { + case __constant_htons(ETH_P_IP): + ip4 = ip_hdr; + if ((void *)ip4 + sizeof(*ip4) > data_end) { + return XDP_DROP; + } + if (ip4->version != 4) { + return XDP_DROP; + } + + /* Check the IP length. Cannot use strict equality due to + * Ethernet padding applied to frames shorter than 64 octects. */ + if (data_end - data < __bpf_ntohs(ip4->tot_len)) { + return XDP_DROP; + } + + if (ip4->frag_off != 0 && + ip4->frag_off != __constant_htons(IP_DF)) { + fragmented = 1; + } + ip_proto = ip4->protocol; + l4_hdr = data + ip4->ihl * 4; + ipv4 = 1; + break; + case __constant_htons(ETH_P_IPV6): + ip6 = ip_hdr; + if ((void *)ip6 + sizeof(*ip6) > data_end) { + return XDP_DROP; + } + if (ip6->version != 6) { + return XDP_DROP; + } + + /* Check the IP length. Cannot use strict equality due to + * Ethernet padding applied to frames shorter than 64 octects. */ + if (data_end - data < __bpf_ntohs(ip6->payload_len) + sizeof(*ip6)) { + return XDP_DROP; + } + + ip_proto = ip6->nexthdr; + data += sizeof(*ip6); + if (ip_proto == IPPROTO_FRAGMENT) { + fragmented = 1; + const struct ipv6_frag_hdr *frag = data; + if ((void *)frag + sizeof(*frag) > data_end) { + return XDP_DROP; + } + ip_proto = frag->nexthdr; + data += sizeof(*frag); + } + l4_hdr = data; + ipv4 = 0; + break; + default: + /* Pass packets of possible other protocols. */ + return XDP_PASS; + } + + const struct tcphdr *tcp; + const struct udphdr *udp; + __u16 port_dest; + __u8 match = 0; + + /* Check the transport protocol. */ + switch (ip_proto) { + case IPPROTO_TCP: + /* Parse TCP header. */ + tcp = l4_hdr; + if (l4_hdr + sizeof(*tcp) > data_end) { + return XDP_DROP; + } + + port_dest = __bpf_ntohs(tcp->dest); + + if ((opts.flags & KNOT_XDP_FILTER_TCP) && + (port_dest == opts.udp_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.udp_port))) { + match = 1; + } + break; + case IPPROTO_UDP: + /* Parse UDP header. */ + udp = l4_hdr; + if (l4_hdr + sizeof(*udp) > data_end) { + return XDP_DROP; + } + + /* Check the UDP length. */ + if (data_end - (void *)udp < __bpf_ntohs(udp->len)) { + return XDP_DROP; + } + + port_dest = __bpf_ntohs(udp->dest); + + if ((opts.flags & KNOT_XDP_FILTER_UDP) && + (port_dest == opts.udp_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.udp_port))) { + match = 1; + } else if ((opts.flags & KNOT_XDP_FILTER_QUIC) && + (port_dest == opts.quic_port || + ((opts.flags & (KNOT_XDP_FILTER_PASS | KNOT_XDP_FILTER_DROP)) && + port_dest >= opts.quic_port))) { + match = 1; + } + break; + default: + /* Pass packets of possible other protocols. */ + return XDP_PASS; + } + + if (!match) { + /* Pass non-matching packet. */ + return XDP_PASS; + } else if (opts.flags & KNOT_XDP_FILTER_DROP) { + /* Drop matching packet if requested. */ + return XDP_DROP; + } else if (fragmented) { + /* Drop fragmented packet. */ + return XDP_DROP; + } + + /* Take into account routing information. */ + if (opts.flags & KNOT_XDP_FILTER_ROUTE) { + struct bpf_fib_lookup fib = { + .ifindex = 1 /* Loopback. */ + }; + if (ipv4) { + fib.family = AF_INET; + fib.ipv4_src = ip4->daddr; + fib.ipv4_dst = ip4->saddr; + } else { + struct in6_addr *ipv6_src = (struct in6_addr *)fib.ipv6_src; + struct in6_addr *ipv6_dst = (struct in6_addr *)fib.ipv6_dst; + fib.family = AF_INET6; + *ipv6_src = ip6->daddr; + *ipv6_dst = ip6->saddr; + } + + const __u16 *mac_in = (const __u16 *)eth_hdr->h_dest; + const __u16 *mac_out = (const __u16 *)fib.smac; + int ret = bpf_fib_lookup(ctx, &fib, sizeof(fib), BPF_FIB_LOOKUP_DIRECT); + switch (ret) { + case BPF_FIB_LKUP_RET_SUCCESS: + /* Cross-interface answers are handled through normal stack. */ + if (mac_in[0] != mac_out[0] || + mac_in[1] != mac_out[1] || + mac_in[2] != mac_out[2]) { + return XDP_PASS; + } + + /* Store output interface index for later use with VLAN in user space. */ + if (meta != 0) { + meta->out_if_index = fib.ifindex; + } + + /* Update destination MAC for responding. */ + __builtin_memcpy(eth_hdr->h_source, fib.dmac, ETH_ALEN); + break; + case BPF_FIB_LKUP_RET_FWD_DISABLED: /* Disabled forwarding on loopback. */ + return XDP_ABORTED; + case BPF_FIB_LKUP_RET_NO_NEIGH: /* Use normal stack to obtain MAC. */ + return XDP_PASS; + default: + return XDP_DROP; + } + } + + /* Forward the packet to user space. */ + return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, 0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/src/libknot/xdp/bpf-user.c b/src/libknot/xdp/bpf-user.c new file mode 100644 index 0000000..449dffe --- /dev/null +++ b/src/libknot/xdp/bpf-user.c @@ -0,0 +1,319 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <bpf/bpf.h> +#include <linux/if_link.h> +#include <net/if.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "libknot/endian.h" +#include "libknot/error.h" +#include "libknot/xdp/bpf-kernel-obj.h" +#include "libknot/xdp/bpf-user.h" +#include "libknot/xdp/eth.h" +#include "contrib/openbsd/strlcpy.h" + +#define NO_BPF_MAPS 2 + +static inline bool IS_ERR_OR_NULL(const void *ptr) +{ + return (ptr == NULL) || (unsigned long)ptr >= (unsigned long)-4095; +} + +static int prog_load(struct bpf_object **pobj, int *prog_fd) +{ + struct bpf_program *prog, *first_prog = NULL; + struct bpf_object *obj; + + obj = bpf_object__open_mem(bpf_kernel_o, bpf_kernel_o_len, NULL); + if (IS_ERR_OR_NULL(obj)) { + return KNOT_ENOENT; + } + + bpf_object__for_each_program(prog, obj) { + bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); + if (first_prog == NULL) { + first_prog = prog; + } + } + + if (first_prog == NULL) { + bpf_object__close(obj); + return KNOT_ENOENT; + } + + int ret = bpf_object__load(obj); + if (ret != 0) { + bpf_object__close(obj); + return KNOT_EINVAL; + } + + *pobj = obj; + *prog_fd = bpf_program__fd(first_prog); + + return KNOT_EOK; +} + +static int ensure_prog(struct kxsk_iface *iface, bool overwrite, bool generic_xdp) +{ + if (bpf_kernel_o_len < 2) { + return KNOT_ENOTSUP; + } + + /* Use libbpf for extracting BPF byte-code from BPF-ELF object, and + * loading this into the kernel via bpf-syscall. */ + int prog_fd; + int ret = prog_load(&iface->prog_obj, &prog_fd); + if (ret != KNOT_EOK) { + return KNOT_EPROGRAM; + } + + uint32_t flags = 0; + if (!overwrite) { + flags |= XDP_FLAGS_UPDATE_IF_NOEXIST; + } + if (generic_xdp) { + flags |= XDP_FLAGS_SKB_MODE; + } + +#if USE_LIBXDP + ret = bpf_xdp_attach(iface->if_index, prog_fd, flags, NULL); +#else + ret = bpf_set_link_xdp_fd(iface->if_index, prog_fd, flags); +#endif + if (ret != 0) { + close(prog_fd); + } + if (ret == -EBUSY && !overwrite) { // We try accepting the present program. + uint32_t prog_id = 0; +#if USE_LIBXDP + ret = bpf_xdp_query_id(iface->if_index, 0, &prog_id); +#else + ret = bpf_get_link_xdp_id(iface->if_index, &prog_id, 0); +#endif + if (ret == 0 && prog_id != 0) { + ret = prog_fd = bpf_prog_get_fd_by_id(prog_id); + } + } + if (ret < 0) { + return KNOT_EFD; + } else { + return prog_fd; + } +} + +static void unget_bpf_maps(struct kxsk_iface *iface) +{ + if (iface->opts_map_fd >= 0) { + close(iface->opts_map_fd); + } + if (iface->xsks_map_fd >= 0) { + close(iface->xsks_map_fd); + } + iface->opts_map_fd = iface->xsks_map_fd = -1; +} + +/*! + * /brief Get FDs for the two maps and assign them into xsk_info-> fields. + * + * Inspired by xsk_lookup_bpf_maps() from libbpf before qidconf_map elimination. + */ +static int get_bpf_maps(int prog_fd, struct kxsk_iface *iface) +{ + uint32_t *map_ids = calloc(NO_BPF_MAPS, sizeof(*map_ids)); + if (map_ids == NULL) { + return KNOT_ENOMEM; + } + + struct bpf_prog_info prog_info = { + .nr_map_ids = NO_BPF_MAPS, + .map_ids = (__u64)(unsigned long)map_ids, + }; + + uint32_t prog_len = sizeof(struct bpf_prog_info); + int ret = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &prog_len); + if (ret != 0) { + free(map_ids); + return ret; + } + + for (int i = 0; i < NO_BPF_MAPS; ++i) { + int fd = bpf_map_get_fd_by_id(map_ids[i]); + if (fd < 0) { + continue; + } + + struct bpf_map_info map_info = { 0 }; + uint32_t map_len = sizeof(struct bpf_map_info); + ret = bpf_obj_get_info_by_fd(fd, &map_info, &map_len); + if (ret != 0) { + close(fd); + continue; + } + + if (strcmp(map_info.name, "opts_map") == 0) { + iface->opts_map_fd = fd; + continue; + } + + if (strcmp(map_info.name, "xsks_map") == 0) { + iface->xsks_map_fd = fd; + continue; + } + + close(fd); + } + + if (iface->opts_map_fd < 0 || iface->xsks_map_fd < 0) { + unget_bpf_maps(iface); + free(map_ids); + return KNOT_ENOENT; + } + + free(map_ids); + return KNOT_EOK; +} + +int kxsk_socket_start(const struct kxsk_iface *iface, knot_xdp_filter_flag_t flags, + uint16_t udp_port, uint16_t quic_port, struct xsk_socket *xsk) +{ + if (iface == NULL || xsk == NULL) { + return KNOT_EINVAL; + } + + int fd = xsk_socket__fd(xsk); + int ret = bpf_map_update_elem(iface->xsks_map_fd, &iface->if_queue, &fd, 0); + if (ret != 0) { + return ret; + } + + knot_xdp_opts_t opts = { + .flags = flags | KNOT_XDP_FILTER_ON, + .udp_port = udp_port, + .quic_port = quic_port, + }; + + ret = bpf_map_update_elem(iface->opts_map_fd, &iface->if_queue, &opts, 0); + if (ret != 0) { + (void)bpf_map_delete_elem(iface->xsks_map_fd, &iface->if_queue); + } + + return ret; +} + +void kxsk_socket_stop(const struct kxsk_iface *iface) +{ + if (iface == NULL) { + return; + } + + knot_xdp_opts_t opts = { 0 }; + + (void)bpf_map_update_elem(iface->opts_map_fd, &iface->if_queue, &opts, 0); + (void)bpf_map_delete_elem(iface->xsks_map_fd, &iface->if_queue); +} + +int kxsk_iface_new(const char *if_name, unsigned if_queue, knot_xdp_load_bpf_t load_bpf, + bool generic_xdp, struct kxsk_iface **out_iface) +{ + if (if_name == NULL || out_iface == NULL) { + return KNOT_EINVAL; + } + + struct kxsk_iface *iface = calloc(1, sizeof(*iface) + IFNAMSIZ); + if (iface == NULL) { + return KNOT_ENOMEM; + } + iface->if_name = (char *)(iface + 1); + strlcpy((char *)iface->if_name, if_name, IFNAMSIZ); + iface->if_index = if_nametoindex(if_name); + if (iface->if_index == 0) { + free(iface); + return KNOT_EINVAL; + } + iface->if_queue = if_queue; + iface->opts_map_fd = iface->xsks_map_fd = -1; + + int ret; + switch (load_bpf) { + case KNOT_XDP_LOAD_BPF_NEVER: + (void)0; + uint32_t prog_id = 0; +#if USE_LIBXDP + ret = bpf_xdp_query_id(iface->if_index, 0, &prog_id); +#else + ret = bpf_get_link_xdp_id(iface->if_index, &prog_id, 0); +#endif + if (ret == 0) { + if (prog_id == 0) { + ret = KNOT_EPROGRAM; + } else { + ret = bpf_prog_get_fd_by_id(prog_id); + } + } + break; + case KNOT_XDP_LOAD_BPF_ALWAYS_UNLOAD: +#if USE_LIBXDP + (void)bpf_xdp_detach(iface->if_index, 0, NULL); +#else + (void)bpf_set_link_xdp_fd(iface->if_index, -1, 0); +#endif + sleep(1); + // FALLTHROUGH + case KNOT_XDP_LOAD_BPF_ALWAYS: + ret = ensure_prog(iface, true, generic_xdp); + break; + case KNOT_XDP_LOAD_BPF_MAYBE: + ret = ensure_prog(iface, false, generic_xdp); + break; + default: + return KNOT_EINVAL; + } + + if (ret >= 0) { + ret = get_bpf_maps(ret, iface); + } + if (ret < 0) { + free(iface); + return ret; + } + + knot_xdp_mode_t mode = knot_eth_xdp_mode(iface->if_index); + if (mode == KNOT_XDP_MODE_NONE) { + free(iface); + return KNOT_ENOTSUP; + } + + *out_iface = iface; + return KNOT_EOK; +} + +void kxsk_iface_free(struct kxsk_iface *iface) +{ + if (iface == NULL) { + return; + } + + unget_bpf_maps(iface); + + if (iface->prog_obj != NULL) { + bpf_object__close(iface->prog_obj); + } + + free(iface); +} diff --git a/src/libknot/xdp/bpf-user.h b/src/libknot/xdp/bpf-user.h new file mode 100644 index 0000000..37aac61 --- /dev/null +++ b/src/libknot/xdp/bpf-user.h @@ -0,0 +1,140 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP socket interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#if USE_LIBXDP + #include <xdp/xsk.h> +#else + #include <bpf/xsk.h> +#endif + +#include "libknot/xdp/xdp.h" + +struct kxsk_iface { + /*! Interface name. */ + const char *if_name; + /*! Interface name index (derived from ifname). */ + int if_index; + /*! Network card queue id. */ + unsigned if_queue; + + /*! Configuration BPF map file descriptor. */ + int opts_map_fd; + /*! XSK BPF map file descriptor. */ + int xsks_map_fd; + + /*! BPF program object. */ + struct bpf_object *prog_obj; +}; + +struct kxsk_umem { + /*! Fill queue: passing memory frames to kernel - ready to receive. */ + struct xsk_ring_prod fq; + /*! Completion queue: passing memory frames from kernel - after send finishes. */ + struct xsk_ring_cons cq; + /*! Handle internal to libbpf. */ + struct xsk_umem *umem; + + /*! The memory frames. */ + struct umem_frame *frames; + /*! The number of free frames (for TX). */ + uint32_t tx_free_count; + /*! Stack of indices of the free frames (for TX). */ + uint16_t tx_free_indices[]; +}; + +struct knot_xdp_socket { + /*! Receive queue: passing arrived packets from kernel. */ + struct xsk_ring_cons rx; + /*! Transmit queue: passing packets to kernel for sending. */ + struct xsk_ring_prod tx; + /*! Information about memory frames for all the passed packets. */ + struct kxsk_umem *umem; + /*! Handle internal to libbpf. */ + struct xsk_socket *xsk; + + /*! Interface context. */ + const struct kxsk_iface *iface; + + /*! If non-NULL, it's a mocked socket with this send function. */ + int (*send_mock)(struct knot_xdp_socket *, const knot_xdp_msg_t[], uint32_t, uint32_t *); + + /*! The kernel has to be woken up by a syscall indication. */ + bool kernel_needs_wakeup; + + /*! The limit of frame size. */ + unsigned frame_limit; + + /*! Mapping of interface indices to VLAN tags. */ + uint16_t *vlan_map; + uint16_t vlan_map_max; +}; + +/*! + * \brief Set up BPF program and map for one XDP socket. + * + * \param if_name Name of the net iface (e.g. eth0). + * \param if_queue Network card queue id. + * \param load_bpf Insert BPF program into packet processing. + * \param generic_xdp Use generic XDP implementation instead of a native one. + * \param out_iface Output: created interface context. + * + * \return KNOT_E* or -errno + */ +int kxsk_iface_new(const char *if_name, unsigned if_queue, knot_xdp_load_bpf_t load_bpf, + bool generic_xdp, struct kxsk_iface **out_iface); + +/*! + * \brief Unload BPF maps for a socket. + * + * \note This keeps the loaded BPF program. We don't care. + * + * \param iface Interface context to be freed. + */ +void kxsk_iface_free(struct kxsk_iface *iface); + +/*! + * \brief Activate this AF_XDP socket through the BPF maps. + * + * \param iface Interface context. + * \param flags XDP filter configuration flags. + * \param udp_port UDP and/or TCP port to listen on if enabled via \a opts. + * \param quic_port QUIC/UDP port to listen on if enabled via \a opts. + * \param xsk Socket ctx. + * + * \return KNOT_E* or -errno + */ +int kxsk_socket_start(const struct kxsk_iface *iface, knot_xdp_filter_flag_t flags, + uint16_t udp_port, uint16_t quic_port, struct xsk_socket *xsk); + +/*! + * \brief Deactivate this AF_XDP socket through the BPF maps. + * + * \param iface Interface context. + */ +void kxsk_socket_stop(const struct kxsk_iface *iface); + +/*! @} */ diff --git a/src/libknot/xdp/eth.c b/src/libknot/xdp/eth.c new file mode 100644 index 0000000..608eb20 --- /dev/null +++ b/src/libknot/xdp/eth.c @@ -0,0 +1,312 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <bpf/libbpf.h> +#include <errno.h> +#include <ifaddrs.h> +#include <linux/ethtool.h> +#include <linux/if_link.h> +#include <linux/if_vlan.h> +#include <linux/sockios.h> +#include <net/if.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include "contrib/openbsd/strlcpy.h" +#include "contrib/sockaddr.h" +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/errcode.h" +#include "libknot/xdp/eth.h" + +_public_ +int knot_eth_queues(const char *devname) +{ + if (devname == NULL) { + return KNOT_EINVAL; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ethtool_channels ch = { + .cmd = ETHTOOL_GCHANNELS + }; + struct ifreq ifr = { + .ifr_data = (char *)&ch + }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + int ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + if (errno == EOPNOTSUPP) { + ret = 1; + } else { + ret = knot_map_errno(); + } + } else { + if (ch.combined_count == 0) { + ret = 1; + } else { + ret = ch.combined_count; + } + } + + close(fd); + return ret; +} + +_public_ +int knot_eth_rss(const char *devname, knot_eth_rss_conf_t **rss_conf) +{ + if (devname == NULL || rss_conf == NULL) { + return KNOT_EINVAL; + } + + struct ethtool_rxfh *ctx = NULL; + knot_eth_rss_conf_t *out = NULL; + int ret = KNOT_ERROR; + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ethtool_rxfh sizes = { + .cmd = ETHTOOL_GRSSH + }; + struct ifreq ifr = { + .ifr_data = (char *)&sizes + }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + ret = knot_map_errno(); + goto finish; + } + + const unsigned data_size = sizes.indir_size * sizeof(sizes.rss_config[0]) + + sizes.key_size; + + ctx = calloc(1, sizeof(*ctx) + data_size); + if (ctx == NULL) { + ret = KNOT_ENOMEM; + goto finish; + } + ctx->cmd = ETHTOOL_GRSSH; + ctx->indir_size = sizes.indir_size; + ctx->key_size = sizes.key_size; + ifr.ifr_data = (char *)ctx; + + ret = ioctl(fd, SIOCETHTOOL, &ifr); + if (ret != 0) { + ret = knot_map_errno(); + goto finish; + } + + out = calloc(1, sizeof(*out) + data_size); + if (out == NULL) { + ret = KNOT_ENOMEM; + goto finish; + } + + out->table_size = sizes.indir_size; + out->key_size = sizes.key_size; + memcpy(out->data, ctx->rss_config, data_size); + out->mask = out->table_size - 1; +finish: + *rss_conf = out; + + free(ctx); + close(fd); + return ret; +} + +_public_ +int knot_eth_mtu(const char *devname) +{ + if (devname == NULL) { + return KNOT_EINVAL; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + return knot_map_errno(); + } + + struct ifreq ifr = { 0 }; + strlcpy(ifr.ifr_name, devname, IFNAMSIZ); + + int ret = ioctl(fd, SIOCGIFMTU, &ifr); + if (ret != 0) { + if (errno == EOPNOTSUPP) { + ret = KNOT_ENOTSUP; + } else { + ret = knot_map_errno(); + } + } else { + ret = ifr.ifr_mtu; + } + + close(fd); + return ret; +} + +_public_ +int knot_eth_name_from_addr(const struct sockaddr_storage *addr, char *out, + size_t out_len) +{ + if (addr == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct ifaddrs *ifaces = NULL; + if (getifaddrs(&ifaces) != 0) { + return -errno; + } + + size_t matches = 0; + char *match_name = NULL; + + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + const struct sockaddr_storage *ifss = (struct sockaddr_storage *)ifa->ifa_addr; + if (ifss == NULL) { // Observed on interfaces without any address. + continue; + } + + if ((ifss->ss_family == addr->ss_family && sockaddr_is_any(addr)) || + sockaddr_cmp(ifss, addr, true) == 0) { + matches++; + match_name = ifa->ifa_name; + } + } + + if (matches == 1) { + size_t len = strlcpy(out, match_name, out_len); + freeifaddrs(ifaces); + return (len >= out_len) ? KNOT_ESPACE : KNOT_EOK; + } + + freeifaddrs(ifaces); + return matches == 0 ? KNOT_EADDRNOTAVAIL : KNOT_ELIMIT; +} + +_public_ +int knot_eth_vlans(uint16_t *vlan_map[], uint16_t *vlan_map_max) +{ + if (vlan_map == NULL || vlan_map_max == NULL) { + return KNOT_EINVAL; + } + + struct ifaddrs *ifaces = NULL; + if (getifaddrs(&ifaces) != 0) { + return knot_map_errno(); + } + + unsigned map_size = 0; + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_PACKET) { + continue; + } + map_size++; + } + + uint16_t *map = calloc(sizeof(uint16_t), 1 + map_size); // Indexed from 1. + if (map == NULL) { + freeifaddrs(ifaces); + return KNOT_ENOMEM; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + for (struct ifaddrs *ifa = ifaces; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || ifa->ifa_addr->sa_family != AF_PACKET) { + continue; + } + + unsigned if_index = if_nametoindex(ifa->ifa_name); + if (if_index == 0) { + close(fd); + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + struct vlan_ioctl_args ifv = { + .cmd = GET_VLAN_REALDEV_NAME_CMD + }; + strlcpy(ifv.device1, ifa->ifa_name, sizeof(ifv.device1)); + + if (ioctl(fd, SIOCGIFVLAN, &ifv) >= 0) { + memset(&ifv, 0, sizeof(ifv)); + ifv.cmd = GET_VLAN_VID_CMD; + strlcpy(ifv.device1, ifa->ifa_name, sizeof(ifv.device1)); + + if (ioctl(fd, SIOCGIFVLAN, &ifv) < 0) { + close(fd); + free(map); + freeifaddrs(ifaces); + return knot_map_errno(); + } + + map[if_index] = htobe16(ifv.u.VID); + } + } + + close(fd); + freeifaddrs(ifaces); + + *vlan_map = map; + *vlan_map_max = map_size; + + return KNOT_EOK; +} + +_public_ +knot_xdp_mode_t knot_eth_xdp_mode(int if_index) +{ +#if USE_LIBXDP + struct bpf_xdp_query_opts info = { .sz = sizeof(info) }; + int ret = bpf_xdp_query(if_index, 0, &info); +#else + struct xdp_link_info info; + int ret = bpf_get_link_xdp_info(if_index, &info, sizeof(info), 0); +#endif + if (ret != 0) { + return KNOT_XDP_MODE_NONE; + } + + switch (info.attach_mode) { + case XDP_ATTACHED_DRV: + case XDP_ATTACHED_HW: + return KNOT_XDP_MODE_FULL; + case XDP_ATTACHED_SKB: + return KNOT_XDP_MODE_EMUL; + default: + return KNOT_XDP_MODE_NONE; + } +} diff --git a/src/libknot/xdp/eth.h b/src/libknot/xdp/eth.h new file mode 100644 index 0000000..8c5b901 --- /dev/null +++ b/src/libknot/xdp/eth.h @@ -0,0 +1,111 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief Ethernet device info interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stddef.h> +#include <stdint.h> + +#define KNOT_XDP_MAX_MTU 1790 + +/*! + * \brief Get number of combined queues of a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * + * \retval < 0 KNOT_E* if error. + * \retval 1 Default no of queues if the dev does not support. + * \return > 0 Number of queues. + */ +int knot_eth_queues(const char *devname); + +/*! + * \brief Network card RSS configuration. + */ +typedef struct { + size_t table_size; /*!< Size of indirection table in four-bytes. */ + size_t key_size; /*!< Size of the RSS key in bytes. */ + uint32_t mask; /*!< Input mask for accessing the table. */ + uint32_t data[]; /*!< Serialized key and table. */ +} knot_eth_rss_conf_t; + +/*! + * \brief Get RSS configuration of a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * \param rss_conf Output RSS configuration. Must be freed explicitly. + * + * \return KNOT_E* + */ +int knot_eth_rss(const char *devname, knot_eth_rss_conf_t **rss_conf); + +/*! + * \brief Get value of MTU setup on a network interface. + * + * \param devname Name of the ethdev (e.g. eth1). + * + * \retval < 0 KNOT_E* if error. + * \return >= 0 Interface MTU. + */ +int knot_eth_mtu(const char *devname); + +/*! + * \brief Get the corresponding network interface name for the address. + * + * \param addr Address of the interface. + * \param out Output buffer for the interface name. + * \param out_len Size of the output buffer. + * + * \return KNOT_E* + */ +int knot_eth_name_from_addr(const struct sockaddr_storage *addr, char *out, + size_t out_len); + +/*! + * \brief Get the mapping of interface index to VLAN tags. + * + * \param vlan_map Output array of the mappings. + * \param vlan_map_max Maximum interface index allowed. + * + * \return KNOT_E* + */ +int knot_eth_vlans(uint16_t *vlan_map[], uint16_t *vlan_map_max); + +typedef enum { + KNOT_XDP_MODE_NONE, /*!< XDP not available, BPF not loaded, or error. */ + KNOT_XDP_MODE_FULL, /*!< Full XDP support in driver or HW. */ + KNOT_XDP_MODE_EMUL, /*!< Emulated XDP support. */ +} knot_xdp_mode_t; + +/*! + * \brief Return the current XDP mode of a network interface. + * + * \param if_index Index of the interface, output from if_nametoindex(). + * + * \return Current XDP mode. + */ +knot_xdp_mode_t knot_eth_xdp_mode(int if_index); + +/*! @} */ diff --git a/src/libknot/xdp/msg.h b/src/libknot/xdp/msg.h new file mode 100644 index 0000000..380f135 --- /dev/null +++ b/src/libknot/xdp/msg.h @@ -0,0 +1,63 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP message description. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdint.h> +#include <linux/if_ether.h> +#include <netinet/in.h> +#include <sys/uio.h> + +/*! \brief Message flags. */ +typedef enum { + KNOT_XDP_MSG_IPV6 = (1 << 0), /*!< This packet is a IPv6 (IPv4 otherwise). */ + KNOT_XDP_MSG_TCP = (1 << 1), /*!< This packet is a TCP (UDP otherwise). */ + KNOT_XDP_MSG_SYN = (1 << 2), /*!< SYN flag set (TCP only). */ + KNOT_XDP_MSG_ACK = (1 << 3), /*!< ACK flag set (TCP only). */ + KNOT_XDP_MSG_FIN = (1 << 4), /*!< FIN flag set (TCP only). */ + KNOT_XDP_MSG_RST = (1 << 5), /*!< RST flag set (TCP only). */ + KNOT_XDP_MSG_MSS = (1 << 6), /*!< MSS option in TCP header (TCP only). */ + KNOT_XDP_MSG_WSC = (1 << 7), /*!< Window Scale option in TCP header. */ + KNOT_XDP_MSG_VLAN = (1 << 8), /*!< This packet will contain VLAN header. */ +} knot_xdp_msg_flag_t; + +/*! \brief Packet description with src & dst MAC & IP addrs + DNS payload. */ +typedef struct knot_xdp_msg { + struct sockaddr_in6 ip_from; + struct sockaddr_in6 ip_to; + uint8_t eth_from[ETH_ALEN]; + uint8_t eth_to[ETH_ALEN]; + knot_xdp_msg_flag_t flags; + struct iovec payload; + uint32_t seqno; + uint32_t ackno; + uint16_t mss; + uint16_t win; + uint8_t win_scale; + uint8_t ecn; + uint16_t vlan_tci; +} knot_xdp_msg_t; + +/*! @} */ diff --git a/src/libknot/xdp/msg_init.h b/src/libknot/xdp/msg_init.h new file mode 100644 index 0000000..8b96129 --- /dev/null +++ b/src/libknot/xdp/msg_init.h @@ -0,0 +1,74 @@ +/* Copyright (C) 2022 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <stdbool.h> +#include <string.h> + +#include "libknot/xdp/msg.h" +#include "libknot/xdp/tcp.h" +#include "libdnssec/random.h" + +inline static bool empty_msg(const knot_xdp_msg_t *msg) +{ + const unsigned tcp_flags = KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK | + KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_RST; + + return (msg->payload.iov_len == 0 && !(msg->flags & tcp_flags)); +} + +inline static void msg_init_base(knot_xdp_msg_t *msg, knot_xdp_msg_flag_t flags) +{ + memset(msg, 0, sizeof(*msg)); + + msg->flags = flags; +} + +inline static void msg_init(knot_xdp_msg_t *msg, knot_xdp_msg_flag_t flags) +{ + msg_init_base(msg, flags); + + if (flags & KNOT_XDP_MSG_TCP) { + msg->ackno = 0; + msg->seqno = dnssec_random_uint32_t(); + if (flags & KNOT_XDP_MSG_SYN) { + msg->flags |= KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC; + } + } +} + +inline static void msg_init_reply(knot_xdp_msg_t *msg, const knot_xdp_msg_t *query) +{ + msg_init_base(msg, query->flags & (KNOT_XDP_MSG_IPV6 | KNOT_XDP_MSG_TCP | + KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC)); + + memcpy(msg->eth_from, query->eth_to, ETH_ALEN); + memcpy(msg->eth_to, query->eth_from, ETH_ALEN); + + memcpy(&msg->ip_from, &query->ip_to, sizeof(msg->ip_from)); + memcpy(&msg->ip_to, &query->ip_from, sizeof(msg->ip_to)); + + msg->vlan_tci = query->vlan_tci; + + if (msg->flags & KNOT_XDP_MSG_TCP) { + msg->ackno = knot_tcp_next_seqno(query); + msg->seqno = query->ackno; + if (msg->seqno == 0) { + msg->seqno = dnssec_random_uint32_t(); + } + } +} diff --git a/src/libknot/xdp/protocols.h b/src/libknot/xdp/protocols.h new file mode 100644 index 0000000..1a18601 --- /dev/null +++ b/src/libknot/xdp/protocols.h @@ -0,0 +1,457 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <assert.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <string.h> + +#include "libknot/endian.h" +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/msg.h" + +/* Don't fragment flag. */ +#define IP_DF 0x4000 + +#define HDR_8021Q_LEN 4; + +/* + * Following prot_read_*() functions do not check sanity of parsed packet. + * Broken packets have to be dropped by BPF filter prior getting here. + */ + +inline static void *prot_read_udp(void *data, uint16_t *src_port, uint16_t *dst_port) +{ + const struct udphdr *udp = data; + + *src_port = udp->source; + *dst_port = udp->dest; + + return data + sizeof(*udp); +} + +enum { + PROT_TCP_OPT_ENDOP = 0, + PROT_TCP_OPT_NOOP = 1, + PROT_TCP_OPT_MSS = 2, + PROT_TCP_OPT_WSC = 3, // window scale + + PROT_TCP_OPT_LEN_MSS = 4, + PROT_TCP_OPT_LEN_WSC = 3, +}; + +inline static void *prot_read_tcp(void *data, knot_xdp_msg_t *msg, uint16_t *src_port, uint16_t *dst_port) +{ + const struct tcphdr *tcp = data; + + msg->flags |= KNOT_XDP_MSG_TCP; + + if (tcp->syn) { + msg->flags |= KNOT_XDP_MSG_SYN; + } + if (tcp->ack) { + msg->flags |= KNOT_XDP_MSG_ACK; + } + if (tcp->fin) { + msg->flags |= KNOT_XDP_MSG_FIN; + } + if (tcp->rst) { + msg->flags |= KNOT_XDP_MSG_RST; + } + + msg->seqno = be32toh(tcp->seq); + msg->ackno = be32toh(tcp->ack_seq); + msg->win = be16toh(tcp->window); + + *src_port = tcp->source; + *dst_port = tcp->dest; + + uint8_t *opts = data + sizeof(*tcp), *hdr_end = data + tcp->doff * 4; + while (opts < hdr_end) { + if (opts[0] == PROT_TCP_OPT_ENDOP || opts[0] == PROT_TCP_OPT_NOOP) { + opts++; + continue; + } + + if (opts + 1 > hdr_end || opts + opts[1] > hdr_end) { + // Malformed option. + break; + } + + if (opts[0] == PROT_TCP_OPT_MSS && opts[1] == PROT_TCP_OPT_LEN_MSS) { + msg->flags |= KNOT_XDP_MSG_MSS; + memcpy(&msg->mss, &opts[2], sizeof(msg->mss)); + msg->mss = be16toh(msg->mss); + } + + if (opts[0] == PROT_TCP_OPT_WSC && opts[1] == PROT_TCP_OPT_LEN_WSC) { + msg->flags |= KNOT_XDP_MSG_WSC; + msg->win_scale = opts[2]; + } + + opts += opts[1]; + } + + return hdr_end; +} + +inline static void *prot_read_ipv4(void *data, knot_xdp_msg_t *msg, void **data_end) +{ + const struct iphdr *ip4 = data; + + // Conditions ensured by the BPF filter. + assert(ip4->version == 4); + assert(ip4->frag_off == 0 || ip4->frag_off == __constant_htons(IP_DF)); + // IPv4 header checksum is not verified! + + struct sockaddr_in *src = (struct sockaddr_in *)&msg->ip_from; + struct sockaddr_in *dst = (struct sockaddr_in *)&msg->ip_to; + memcpy(&src->sin_addr, &ip4->saddr, sizeof(src->sin_addr)); + memcpy(&dst->sin_addr, &ip4->daddr, sizeof(dst->sin_addr)); + src->sin_family = AF_INET; + dst->sin_family = AF_INET; + + msg->ecn = (ip4->tos & 0x3); + + *data_end = data + be16toh(ip4->tot_len); + data += ip4->ihl * 4; + + if (ip4->protocol == IPPROTO_TCP) { + return prot_read_tcp(data, msg, &src->sin_port, &dst->sin_port); + } else { + assert(ip4->protocol == IPPROTO_UDP); + return prot_read_udp(data, &src->sin_port, &dst->sin_port); + } +} + +inline static void *prot_read_ipv6(void *data, knot_xdp_msg_t *msg, void **data_end) +{ + const struct ipv6hdr *ip6 = data; + + msg->flags |= KNOT_XDP_MSG_IPV6; + + // Conditions ensured by the BPF filter. + assert(ip6->version == 6); + + struct sockaddr_in6 *src = (struct sockaddr_in6 *)&msg->ip_from; + struct sockaddr_in6 *dst = (struct sockaddr_in6 *)&msg->ip_to; + memcpy(&src->sin6_addr, &ip6->saddr, sizeof(src->sin6_addr)); + memcpy(&dst->sin6_addr, &ip6->daddr, sizeof(dst->sin6_addr)); + src->sin6_family = AF_INET6; + dst->sin6_family = AF_INET6; + src->sin6_flowinfo = 0; + dst->sin6_flowinfo = 0; + // Scope ID is ignored. + + msg->ecn = ((ip6->flow_lbl[0] & 0x30) >> 4); + + data += sizeof(*ip6); + *data_end = data + be16toh(ip6->payload_len); + + if (ip6->nexthdr == IPPROTO_TCP) { + return prot_read_tcp(data, msg, &src->sin6_port, &dst->sin6_port); + } else { + assert(ip6->nexthdr == IPPROTO_UDP); + return prot_read_udp(data, &src->sin6_port, &dst->sin6_port); + } +} + +inline static void *prot_read_eth(void *data, knot_xdp_msg_t *msg, void **data_end, + const uint16_t *vlan_map, unsigned vlan_map_max) +{ + const struct ethhdr *eth = data; + knot_xdp_info_t *info = data - KNOT_XDP_PKT_ALIGNMENT - sizeof(*info); + + memcpy(msg->eth_from, eth->h_source, ETH_ALEN); + memcpy(msg->eth_to, eth->h_dest, ETH_ALEN); + msg->flags = 0; + + if (eth->h_proto == __constant_htons(ETH_P_8021Q)) { + if (info->out_if_index > 0 && info->out_if_index <= vlan_map_max) { + assert(vlan_map); + msg->vlan_tci = vlan_map[info->out_if_index]; + } else { + memcpy(&msg->vlan_tci, data + sizeof(*eth), sizeof(msg->vlan_tci)); + } + data += HDR_8021Q_LEN; + eth = data; + } + + data += sizeof(*eth); + + if (eth->h_proto == __constant_htons(ETH_P_IPV6)) { + return prot_read_ipv6(data, msg, data_end); + } else { + assert(eth->h_proto == __constant_htons(ETH_P_IP)); + return prot_read_ipv4(data, msg, data_end); + } +} + +inline static size_t prot_write_hdrs_len(const knot_xdp_msg_t *msg) +{ + size_t res = sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr); + + if (msg->vlan_tci != 0 || msg->flags & KNOT_XDP_MSG_VLAN) { + res += HDR_8021Q_LEN; + } + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + res += sizeof(struct ipv6hdr) - sizeof(struct iphdr); + } + + if (msg->flags & KNOT_XDP_MSG_TCP) { + res += sizeof(struct tcphdr) - sizeof(struct udphdr); + + if (msg->flags & KNOT_XDP_MSG_MSS) { + res += PROT_TCP_OPT_LEN_MSS; + } + if (msg->flags & KNOT_XDP_MSG_WSC) { + res += PROT_TCP_OPT_LEN_WSC + 1; // 1 == align + } + } + + return res; +} + +/* Checksum endianness implementation notes for ipv4_checksum() and checksum(). + * + * The basis for checksum is addition on big-endian 16-bit words, with bit 16 carrying + * over to bit 0. That can be viewed as first byte carrying to the second and the + * second one carrying back to the first one, i.e. a symmetrical situation. + * Therefore the result is the same even when arithmetics is done on little-endian (!) + */ + +inline static void checksum(uint32_t *result, const void *_data, uint32_t _data_len) +{ + assert(!(_data_len & 1)); + const uint16_t *data = _data; + uint32_t len = _data_len / 2; + while (len-- > 0) { + *result += *data++; + } +} + +inline static void checksum_uint16(uint32_t *result, uint16_t x) +{ + checksum(result, &x, sizeof(x)); +} + +inline static void checksum_payload(uint32_t *result, void *payload, size_t pay_len) +{ + if (pay_len & 1) { + ((uint8_t *)payload)[pay_len++] = 0; + } + checksum(result, payload, pay_len); +} + +inline static uint16_t checksum_finish(uint32_t result, bool nonzero) +{ + while (result > 0xffff) { + result = (result & 0xffff) + (result >> 16); + } + if (!nonzero || result != 0xffff) { + result = ~result; + } + return result; +} + +inline static void prot_write_udp(void *data, const knot_xdp_msg_t *msg, void *data_end, + uint16_t src_port, uint16_t dst_port, uint32_t chksum) +{ + struct udphdr *udp = data; + + udp->len = htobe16(data_end - data); + udp->source = src_port; + udp->dest = dst_port; + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + udp->check = 0; + checksum(&chksum, &udp->len, sizeof(udp->len)); + checksum_uint16(&chksum, htobe16(IPPROTO_UDP)); + checksum_payload(&chksum, data, data_end - data); + udp->check = checksum_finish(chksum, true); + } else { + udp->check = 0; // UDP over IPv4 doesn't require checksum. + } + + assert(data + sizeof(*udp) == msg->payload.iov_base); +} + +inline static void prot_write_tcp(void *data, const knot_xdp_msg_t *msg, void *data_end, + uint16_t src_port, uint16_t dst_port, uint32_t chksum, + uint16_t mss) +{ + struct tcphdr *tcp = data; + + tcp->source = src_port; + tcp->dest = dst_port; + tcp->seq = htobe32(msg->seqno); + tcp->ack_seq = htobe32(msg->ackno); + tcp->window = htobe16(msg->win); + tcp->check = 0; // Temporarily initialize before checksum calculation. + + tcp->res1 = 0; + tcp->urg = 0; + tcp->ece = 0; + tcp->cwr = 0; + tcp->urg_ptr = 0; + + tcp->syn = ((msg->flags & KNOT_XDP_MSG_SYN) ? 1 : 0); + tcp->ack = ((msg->flags & KNOT_XDP_MSG_ACK) ? 1 : 0); + tcp->fin = ((msg->flags & KNOT_XDP_MSG_FIN) ? 1 : 0); + tcp->rst = ((msg->flags & KNOT_XDP_MSG_RST) ? 1 : 0); + + uint8_t *hdr_end = data + sizeof(*tcp); + if (msg->flags & KNOT_XDP_MSG_WSC) { + hdr_end[0] = PROT_TCP_OPT_WSC; + hdr_end[1] = PROT_TCP_OPT_LEN_WSC; + hdr_end[2] = msg->win_scale; + hdr_end += PROT_TCP_OPT_LEN_WSC; + *hdr_end++ = PROT_TCP_OPT_NOOP; // align + } + if (msg->flags & KNOT_XDP_MSG_MSS) { + mss = htobe16(mss); + hdr_end[0] = PROT_TCP_OPT_MSS; + hdr_end[1] = PROT_TCP_OPT_LEN_MSS; + memcpy(&hdr_end[2], &mss, sizeof(mss)); + hdr_end += PROT_TCP_OPT_LEN_MSS; + } + + tcp->psh = ((data_end - (void *)hdr_end > 0) ? 1 : 0); + tcp->doff = (hdr_end - (uint8_t *)tcp) / 4; + assert((hdr_end - (uint8_t *)tcp) % 4 == 0); + + checksum_uint16(&chksum, htobe16(IPPROTO_TCP)); + checksum_uint16(&chksum, htobe16(data_end - data)); + checksum_payload(&chksum, data, data_end - data); + tcp->check = checksum_finish(chksum, false); + + assert(hdr_end == msg->payload.iov_base); +} + +inline static uint16_t from32to16(uint32_t sum) +{ + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return sum; +} + +inline static uint16_t ipv4_checksum(const uint16_t *ipv4_hdr) +{ + uint32_t sum32 = 0; + for (int i = 0; i < 10; ++i) { + if (i != 5) { + sum32 += ipv4_hdr[i]; + } + } + return ~from32to16(sum32); +} + +inline static void prot_write_ipv4(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct iphdr *ip4 = data; + + ip4->version = 4; + ip4->ihl = sizeof(*ip4) / 4; + ip4->tos = msg->ecn; + ip4->tot_len = htobe16(data_end - data); + ip4->id = 0; + ip4->frag_off = 0; + ip4->ttl = IPDEFTTL; + ip4->protocol = ((msg->flags & KNOT_XDP_MSG_TCP) ? IPPROTO_TCP : IPPROTO_UDP); + + const struct sockaddr_in *src = (const struct sockaddr_in *)&msg->ip_from; + const struct sockaddr_in *dst = (const struct sockaddr_in *)&msg->ip_to; + memcpy(&ip4->saddr, &src->sin_addr, sizeof(src->sin_addr)); + memcpy(&ip4->daddr, &dst->sin_addr, sizeof(dst->sin_addr)); + + ip4->check = ipv4_checksum(data); + + data += sizeof(*ip4); + + if (msg->flags & KNOT_XDP_MSG_TCP) { + uint32_t chk = 0; + checksum(&chk, &src->sin_addr, sizeof(src->sin_addr)); + checksum(&chk, &dst->sin_addr, sizeof(dst->sin_addr)); + + prot_write_tcp(data, msg, data_end, src->sin_port, dst->sin_port, chk, tcp_mss); + } else { + prot_write_udp(data, msg, data_end, src->sin_port, dst->sin_port, 0); // IPv4/UDP requires no checksum + } +} + +inline static void prot_write_ipv6(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct ipv6hdr *ip6 = data; + + ip6->version = 6; + ip6->priority = 0; + ip6->flow_lbl[0] = (msg->ecn << 4); + ip6->payload_len = htobe16(data_end - data - sizeof(*ip6)); + ip6->nexthdr = ((msg->flags & KNOT_XDP_MSG_TCP) ? IPPROTO_TCP : IPPROTO_UDP); + ip6->hop_limit = IPDEFTTL; + + memset(ip6->flow_lbl, 0, sizeof(ip6->flow_lbl)); + + const struct sockaddr_in6 *src = (const struct sockaddr_in6 *)&msg->ip_from; + const struct sockaddr_in6 *dst = (const struct sockaddr_in6 *)&msg->ip_to; + memcpy(&ip6->saddr, &src->sin6_addr, sizeof(src->sin6_addr)); + memcpy(&ip6->daddr, &dst->sin6_addr, sizeof(dst->sin6_addr)); + + data += sizeof(*ip6); + + uint32_t chk = 0; + checksum(&chk, &src->sin6_addr, sizeof(src->sin6_addr)); + checksum(&chk, &dst->sin6_addr, sizeof(dst->sin6_addr)); + + if (msg->flags & KNOT_XDP_MSG_TCP) { + prot_write_tcp(data, msg, data_end, src->sin6_port, dst->sin6_port, chk, tcp_mss); + } else { + prot_write_udp(data, msg, data_end, src->sin6_port, dst->sin6_port, chk); + } +} + +inline static void prot_write_eth(void *data, const knot_xdp_msg_t *msg, + void *data_end, uint16_t tcp_mss) +{ + struct ethhdr *eth = data; + + memcpy(eth->h_source, msg->eth_from, ETH_ALEN); + memcpy(eth->h_dest, msg->eth_to, ETH_ALEN); + + if (msg->vlan_tci != 0) { + eth->h_proto = __constant_htons(ETH_P_8021Q); + memcpy(data + sizeof(*eth), &msg->vlan_tci, sizeof(msg->vlan_tci)); + data += HDR_8021Q_LEN; + eth = data; + } + + data += sizeof(*eth); + + if (msg->flags & KNOT_XDP_MSG_IPV6) { + eth->h_proto = __constant_htons(ETH_P_IPV6); + prot_write_ipv6(data, msg, data_end, tcp_mss); + } else { + eth->h_proto = __constant_htons(ETH_P_IP); + prot_write_ipv4(data, msg, data_end, tcp_mss); + } +} diff --git a/src/libknot/xdp/tcp.c b/src/libknot/xdp/tcp.c new file mode 100644 index 0000000..7d647d7 --- /dev/null +++ b/src/libknot/xdp/tcp.c @@ -0,0 +1,749 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "libknot/xdp/tcp.h" +#include "libknot/xdp/tcp_iobuf.h" +#include "libknot/attribute.h" +#include "libknot/error.h" +#include "libdnssec/random.h" +#include "contrib/macros.h" +#include "contrib/openbsd/siphash.h" +#include "contrib/ucw/lists.h" + +static uint32_t get_timestamp(void) +{ + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t res = (uint64_t)t.tv_sec * 1000000; + res += (uint64_t)t.tv_nsec / 1000; + return res & 0xffffffff; // overflow does not matter since we are working with differences +} + +static size_t sockaddr_data_len(const struct sockaddr_in6 *rem, const struct sockaddr_in6 *loc) +{ + assert(rem->sin6_family == loc->sin6_family); + if (rem->sin6_family == AF_INET) { + return offsetof(struct sockaddr_in, sin_zero); + } else { + assert(rem->sin6_family == AF_INET6); + return offsetof(struct sockaddr_in6, sin6_scope_id); + } +} + +static uint64_t hash_four_tuple(const struct sockaddr_in6 *rem, const struct sockaddr_in6 *loc, + knot_tcp_table_t *table) +{ + size_t socka_data_len = sockaddr_data_len(rem, loc); + SIPHASH_CTX ctx; + SipHash24_Init(&ctx, (const SIPHASH_KEY *)(table->hash_secret)); + SipHash24_Update(&ctx, rem, socka_data_len); + SipHash24_Update(&ctx, loc, socka_data_len); + return SipHash24_End(&ctx); +} + +static list_t *tcp_table_timeout(knot_tcp_table_t *table) +{ + return (list_t *)&table->conns[table->size]; +} + +static node_t *tcp_conn_node(knot_tcp_conn_t *conn) +{ + return (node_t *)&conn->list_node_placeholder; +} + +static bool conn_removed(knot_tcp_conn_t *conn) +{ + return tcp_conn_node(conn)->next == NULL; +} + +static void next_node_ptr(knot_tcp_conn_t **ptr) +{ + if (*ptr != NULL) { + assert(!conn_removed(*ptr)); + + *ptr = (*ptr)->list_node_placeholder.list_node_next; + if ((*ptr)->list_node_placeholder.list_node_next == NULL) { // detected tail of list + *ptr = NULL; + } + } +} + +static void next_ptr_ibuf(knot_tcp_conn_t **ptr) +{ + do { + next_node_ptr(ptr); + } while (*ptr != NULL && (*ptr)->inbuf.iov_len == 0); +} + +static void next_ptr_obuf(knot_tcp_conn_t **ptr) +{ + do { + next_node_ptr(ptr); + } while (*ptr != NULL && knot_tcp_outbufs_usage((*ptr)->outbufs) == 0); +} + +_public_ +knot_tcp_table_t *knot_tcp_table_new(size_t size, knot_tcp_table_t *secret_share) +{ + knot_tcp_table_t *table = calloc(1, sizeof(*table) + sizeof(list_t) + + size * sizeof(table->conns[0])); + if (table == NULL) { + return table; + } + + table->size = size; + init_list(tcp_table_timeout(table)); + + assert(sizeof(table->hash_secret) == sizeof(SIPHASH_KEY)); + if (secret_share == NULL) { + table->hash_secret[0] = dnssec_random_uint64_t(); + table->hash_secret[1] = dnssec_random_uint64_t(); + } else { + table->hash_secret[0] = secret_share->hash_secret[0]; + table->hash_secret[1] = secret_share->hash_secret[1]; + } + + return table; +} + +static void del_conn(knot_tcp_conn_t *conn) +{ + if (conn != NULL) { + free(conn->inbuf.iov_base); + while (conn->outbufs != NULL) { + struct knot_tcp_outbuf *next = conn->outbufs->next; + free(conn->outbufs); + conn->outbufs = next; + } + free(conn); + } +} + +_public_ +void knot_tcp_table_free(knot_tcp_table_t *table) +{ + if (table != NULL) { + knot_tcp_conn_t *conn, *next; + WALK_LIST_DELSAFE(conn, next, *tcp_table_timeout(table)) { + del_conn(conn); + } + free(table); + } +} + +static knot_tcp_conn_t **tcp_table_lookup(const struct sockaddr_in6 *rem, + const struct sockaddr_in6 *loc, + uint64_t *hash, knot_tcp_table_t *table) +{ + if (*hash == 0) { + *hash = hash_four_tuple(rem, loc, table); + } + size_t sdl = sockaddr_data_len(rem, loc); + knot_tcp_conn_t **res = table->conns + (*hash % table->size); + while (*res != NULL) { + if (memcmp(&(*res)->ip_rem, rem, sdl) == 0 && + memcmp(&(*res)->ip_loc, loc, sdl) == 0) { + break; + } + res = &(*res)->next; + } + return res; +} + +static knot_tcp_conn_t **tcp_table_re_lookup(knot_tcp_conn_t *conn, + knot_tcp_table_t *table) +{ + uint64_t unused_hash = 0; + knot_tcp_conn_t **res = tcp_table_lookup(&conn->ip_rem, &conn->ip_loc, + &unused_hash, table); + assert(*res == conn); + return res; +} + +static void rem_align_pointers(knot_tcp_conn_t *to_rem, knot_tcp_table_t *table) +{ + assert(!conn_removed(to_rem)); + if (to_rem == table->next_close) { + next_node_ptr(&table->next_close); + } + if (to_rem == table->next_ibuf) { + next_ptr_ibuf(&table->next_ibuf); + } + if (to_rem == table->next_obuf) { + next_ptr_obuf(&table->next_obuf); + } + if (to_rem == table->next_resend) { + next_ptr_obuf(&table->next_resend); + } +} + +static void tcp_table_remove_conn(knot_tcp_conn_t **todel) +{ + rem_node(tcp_conn_node(*todel)); // remove from timeout double-linked list + *todel = (*todel)->next; // remove from conn-table linked list +} + +static void tcp_table_remove(knot_tcp_conn_t **todel, knot_tcp_table_t *table) +{ + assert(table->usage > 0); + rem_align_pointers(*todel, table); + table->inbufs_total -= buffer_alloc_size((*todel)->inbuf.iov_len); + table->outbufs_total -= knot_tcp_outbufs_usage((*todel)->outbufs); + tcp_table_remove_conn(todel); + table->usage--; +} + +static void conn_init_from_msg(knot_tcp_conn_t *conn, knot_xdp_msg_t *msg) +{ + memcpy(&conn->ip_rem, &msg->ip_from, sizeof(conn->ip_rem)); + memcpy(&conn->ip_loc, &msg->ip_to, sizeof(conn->ip_loc)); + + memcpy(&conn->last_eth_rem, &msg->eth_from, sizeof(conn->last_eth_rem)); + memcpy(&conn->last_eth_loc, &msg->eth_to, sizeof(conn->last_eth_loc)); + + conn->seqno = msg->seqno; + conn->ackno = msg->ackno; + conn->acked = msg->ackno; + + conn->last_active = get_timestamp(); + conn->state = XDP_TCP_NORMAL; + conn->establish_rtt = 0; + + memset(&conn->inbuf, 0, sizeof(conn->inbuf)); + memset(&conn->outbufs, 0, sizeof(conn->outbufs)); +} + +static void tcp_table_insert(knot_tcp_conn_t *conn, uint64_t hash, + knot_tcp_table_t *table) +{ + knot_tcp_conn_t **addto = table->conns + (hash % table->size); + add_tail(tcp_table_timeout(table), tcp_conn_node(conn)); + if (table->next_close == NULL) { + table->next_close = conn; + } + conn->next = *addto; + *addto = conn; + table->usage++; +} + +// WARNING you shall ensure that it's not in the table already! +static int tcp_table_add(knot_xdp_msg_t *msg, uint64_t hash, knot_tcp_table_t *table, + knot_tcp_conn_t **res) +{ + knot_tcp_conn_t *c = malloc(sizeof(*c)); + if (c == NULL) { + return KNOT_ENOMEM; + } + conn_init_from_msg(c, msg); + tcp_table_insert(c, hash, table); + *res = c; + return KNOT_EOK; +} + +static bool check_seq_ack(const knot_xdp_msg_t *msg, const knot_tcp_conn_t *conn) +{ + if (conn == NULL || conn->seqno != msg->seqno) { + return false; + } + + if (conn->acked <= conn->ackno) { // ackno does not wrap around uint32 + return (msg->ackno >= conn->acked && msg->ackno <= conn->ackno); + } else { // this is more tricky + return (msg->ackno >= conn->acked || msg->ackno <= conn->ackno); + } +} + +static void conn_update(knot_tcp_conn_t *conn, const knot_xdp_msg_t *msg) +{ + conn->seqno = knot_tcp_next_seqno(msg); + memcpy(conn->last_eth_rem, msg->eth_from, sizeof(conn->last_eth_rem)); + memcpy(conn->last_eth_loc, msg->eth_to, sizeof(conn->last_eth_loc)); + conn->window_size = (uint32_t)msg->win * (1LU << conn->window_scale); + + uint32_t now = get_timestamp(); + if (conn->establish_rtt == 0 && conn->last_active != 0) { + conn->establish_rtt = now - conn->last_active; + } + conn->last_active = now; +} + +_public_ +int knot_tcp_recv(knot_tcp_relay_t *relays, knot_xdp_msg_t msgs[], uint32_t msg_count, + knot_tcp_table_t *tcp_table, knot_tcp_table_t *syn_table, + knot_tcp_ignore_t ignore) +{ + if (msg_count == 0) { + return KNOT_EOK; + } + if (relays == NULL || msgs == NULL || tcp_table == NULL) { + return KNOT_EINVAL; + } + memset(relays, 0, msg_count * sizeof(*relays)); + + knot_tcp_relay_t *relay = relays; + int ret = KNOT_EOK; + + for (knot_xdp_msg_t *msg = msgs; msg != msgs + msg_count && ret == KNOT_EOK; msg++) { + if (!(msg->flags & KNOT_XDP_MSG_TCP)) { + continue; + } + + uint64_t conn_hash = 0; + knot_tcp_conn_t **pconn = tcp_table_lookup(&msg->ip_from, &msg->ip_to, + &conn_hash, tcp_table); + knot_tcp_conn_t *conn = *pconn; + bool seq_ack_match = check_seq_ack(msg, conn); + if (seq_ack_match) { + assert(conn->mss != 0); + conn_update(conn, msg); + + rem_align_pointers(conn, tcp_table); + rem_node(tcp_conn_node(conn)); + add_tail(tcp_table_timeout(tcp_table), tcp_conn_node(conn)); + + if (msg->flags & KNOT_XDP_MSG_ACK) { + conn->acked = msg->ackno; + knot_tcp_outbufs_ack(&conn->outbufs, msg->ackno, &tcp_table->outbufs_total); + } + } + + relay->msg = msg; + relay->conn = conn; + + // process incoming data + if (seq_ack_match && (msg->flags & KNOT_XDP_MSG_ACK) && msg->payload.iov_len > 0) { + if (!(ignore & XDP_TCP_IGNORE_DATA_ACK)) { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + ret = knot_tcp_inbufs_upd(&conn->inbuf, msg->payload, false, + &relay->inbf, &tcp_table->inbufs_total); + if (ret != KNOT_EOK) { + break; + } + if (conn->inbuf.iov_len > 0 && tcp_table->next_ibuf == NULL) { + tcp_table->next_ibuf = conn; + } + } + + // process TCP connection state + switch (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK | + KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_RST)) { + case KNOT_XDP_MSG_SYN: + case (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK): + if (conn == NULL) { + bool synack = (msg->flags & KNOT_XDP_MSG_ACK); + + knot_tcp_table_t *add_table = tcp_table; + if (syn_table != NULL && !synack) { + add_table = syn_table; + if (*tcp_table_lookup(&msg->ip_from, &msg->ip_to, &conn_hash, syn_table) != NULL) { + break; + } + } + + ret = tcp_table_add(msg, conn_hash, add_table, &relay->conn); + if (ret == KNOT_EOK) { + relay->action = synack ? XDP_TCP_ESTABLISH : XDP_TCP_SYN; + if (!(ignore & XDP_TCP_IGNORE_ESTABLISH)) { + relay->auto_answer = synack ? KNOT_XDP_MSG_ACK : (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_ACK); + } + + conn = relay->conn; + conn->state = synack ? XDP_TCP_NORMAL: XDP_TCP_ESTABLISHING; + conn->mss = MAX(msg->mss, 536); // minimal MSS, most importantly not zero! + conn->window_scale = msg->win_scale; + conn_update(conn, msg); + if (!synack) { + conn->acked = dnssec_random_uint32_t(); + conn->ackno = conn->acked; + } + } + } else { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + break; + case KNOT_XDP_MSG_ACK: + if (!seq_ack_match) { + if (syn_table != NULL && msg->payload.iov_len == 0 && + (pconn = tcp_table_lookup(&msg->ip_from, &msg->ip_to, &conn_hash, syn_table)) != NULL && + (conn = *pconn) != NULL && check_seq_ack(msg, conn)) { + // move conn from syn_table to tcp_table + tcp_table_remove(pconn, syn_table); + tcp_table_insert(conn, conn_hash, tcp_table); + relay->conn = conn; + relay->action = XDP_TCP_ESTABLISH; + conn->state = XDP_TCP_NORMAL; + conn_update(conn, msg); + } + } else { + switch (conn->state) { + case XDP_TCP_NORMAL: + case XDP_TCP_CLOSING1: // just a mess, ignore + break; + case XDP_TCP_ESTABLISHING: + conn->state = XDP_TCP_NORMAL; + relay->action = XDP_TCP_ESTABLISH; + break; + case XDP_TCP_CLOSING2: + if (msg->payload.iov_len == 0) { // otherwise ignore close + tcp_table_remove(pconn, tcp_table); + relay->answer = XDP_TCP_FREE; + } + break; + } + } + break; + case (KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK): + if (ignore & XDP_TCP_IGNORE_FIN) { + break; + } + if (!seq_ack_match) { + if (conn != NULL) { + relay->auto_answer = KNOT_XDP_MSG_RST; + relay->auto_seqno = msg->ackno; + } // else ignore. It would be better and possible, but no big value for the price of CPU. + } else { + if (conn->state == XDP_TCP_CLOSING1) { + relay->action = XDP_TCP_CLOSE; + relay->auto_answer = KNOT_XDP_MSG_ACK; + relay->answer = XDP_TCP_FREE; + tcp_table_remove(pconn, tcp_table); + } else if (msg->payload.iov_len == 0) { // otherwise ignore FIN + relay->action = XDP_TCP_CLOSE; + relay->auto_answer = KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK; + conn->state = XDP_TCP_CLOSING2; + } + } + break; + case KNOT_XDP_MSG_RST: + if (conn != NULL && msg->seqno == conn->seqno) { + relay->action = XDP_TCP_RESET; + tcp_table_remove(pconn, tcp_table); + relay->answer = XDP_TCP_FREE; + } else if (conn != NULL) { + relay->auto_answer = KNOT_XDP_MSG_ACK; + } + break; + default: + break; + } + + if (!knot_tcp_relay_empty(relay)) { + relay++; + } + } + + return ret; +} + +_public_ +int knot_tcp_reply_data(knot_tcp_relay_t *relay, knot_tcp_table_t *tcp_table, + bool ignore_lastbyte, uint8_t *data, uint32_t len) +{ + if (relay == NULL || tcp_table == NULL || relay->conn == NULL || conn_removed(relay->conn)) { + return KNOT_EINVAL; + } + int ret = knot_tcp_outbufs_add(&relay->conn->outbufs, data, len, ignore_lastbyte, + relay->conn->mss, &tcp_table->outbufs_total); + + if (tcp_table->next_obuf == NULL && knot_tcp_outbufs_usage(relay->conn->outbufs) > 0) { + tcp_table->next_obuf = relay->conn; + } + if (tcp_table->next_resend == NULL && knot_tcp_outbufs_usage(relay->conn->outbufs) > 0) { + tcp_table->next_resend = relay->conn; + } + return ret; +} + +static knot_xdp_msg_t *first_msg(knot_xdp_msg_t *msgs, uint32_t n_msgs) +{ + memset(msgs, 0, n_msgs * sizeof(*msgs)); + return msgs - 1; // will be incremented just before first use +} + +static int send_msgs(knot_xdp_msg_t *msgs, uint32_t n_msgs, knot_xdp_socket_t *socket) +{ + assert(socket); + assert(msgs); + + if (n_msgs > 0) { + uint32_t unused; + return knot_xdp_send(socket, msgs, n_msgs, &unused); + } + + return KNOT_EOK; +} + +static void msg_init_from_conn(knot_xdp_msg_t *msg, knot_tcp_conn_t *conn) +{ + memcpy( msg->eth_from, conn->last_eth_loc, sizeof(msg->eth_from)); + memcpy( msg->eth_to, conn->last_eth_rem, sizeof(msg->eth_to)); + memcpy(&msg->ip_from, &conn->ip_loc, sizeof(msg->ip_from)); + memcpy(&msg->ip_to, &conn->ip_rem, sizeof(msg->ip_to)); + + msg->ackno = conn->seqno; + msg->seqno = conn->ackno; + + msg->payload.iov_len = 0; + + msg->win_scale = 14; // maximum possible + msg->win = 0xffff; +} + +static int next_msg(knot_xdp_msg_t *msgs, uint32_t n_msgs, knot_xdp_msg_t **cur, + knot_xdp_socket_t *socket, knot_tcp_relay_t *rl) +{ + (*cur)++; + if (*cur - msgs >= n_msgs) { + int ret = send_msgs(msgs, n_msgs, socket); + if (ret != KNOT_EOK) { + return ret; + } + *cur = first_msg(msgs, n_msgs); + (*cur)++; + } + + knot_xdp_msg_t *msg = *cur; + + knot_xdp_msg_flag_t fl = KNOT_XDP_MSG_TCP; + if (rl->conn->ip_loc.sin6_family == AF_INET6) { + fl |= KNOT_XDP_MSG_IPV6; + } + if (rl->conn->state == XDP_TCP_ESTABLISHING) { + fl |= KNOT_XDP_MSG_MSS | KNOT_XDP_MSG_WSC; + } + + int ret = knot_xdp_send_alloc(socket, fl, msg); + if (ret != KNOT_EOK) { + return ret; + } + + msg_init_from_conn(msg, rl->conn); + + return ret; +} + +_public_ +int knot_tcp_send(knot_xdp_socket_t *socket, knot_tcp_relay_t relays[], + uint32_t relay_count, uint32_t max_at_once) +{ + if (relay_count == 0) { + return KNOT_EOK; + } + if (socket == NULL || relays == NULL) { + return KNOT_EINVAL; + } + + knot_xdp_msg_t msgs[max_at_once], *first = first_msg(msgs, max_at_once), *msg = first; + + for (uint32_t i = 0; i < relay_count; i++) { + knot_tcp_relay_t *rl = &relays[i]; + +#define NEXT_MSG { \ + int ret = next_msg(msgs, max_at_once, &msg, socket, rl); \ + if (ret != KNOT_EOK) { return ret; } \ +} + + if (rl->auto_answer != 0) { + NEXT_MSG + msg->flags |= rl->auto_answer; + if (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_FIN)) { + rl->conn->ackno++; + } + if (rl->auto_answer == KNOT_XDP_MSG_RST) { + msg->seqno = rl->auto_seqno; + } + } + + switch (rl->answer & 0x0f) { + case XDP_TCP_ESTABLISH: + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_SYN; + rl->conn->ackno++; + break; + case XDP_TCP_CLOSE: + NEXT_MSG + msg->flags |= (KNOT_XDP_MSG_FIN | KNOT_XDP_MSG_ACK); + rl->conn->ackno++; + rl->conn->state = XDP_TCP_CLOSING1; + break; + case XDP_TCP_RESET: + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_RST; + break; + case XDP_TCP_NOOP: + default: + break; + } + + size_t can_data = 0; + knot_tcp_outbuf_t *ob; + if (rl->conn != NULL) { + knot_tcp_outbufs_can_send(rl->conn->outbufs, rl->conn->window_size, + rl->answer == XDP_TCP_RESEND, &ob, &can_data); + } + while (can_data > 0) { + NEXT_MSG + msg->flags |= KNOT_XDP_MSG_ACK; + msg->payload.iov_len = ob->len; + memcpy(msg->payload.iov_base, ob->bytes, ob->len); + + if (!ob->sent) { + assert(rl->conn->ackno == msg->seqno); + rl->conn->ackno += msg->payload.iov_len; + } else { + msg->seqno = ob->seqno; + } + + ob->sent = true; + ob->seqno = msg->seqno; + + can_data--; + ob = ob->next; + } +#undef NEXT_MSG + } + + return send_msgs(msgs, msg - first, socket); +} + +static void sweep_reset(knot_tcp_table_t *tcp_table, knot_tcp_relay_t *rl, + ssize_t *free_conns, ssize_t *free_inbuf, ssize_t *free_outbuf, + knot_sweep_stats_t *stats, knot_sweep_counter_t counter) +{ + rl->answer = XDP_TCP_RESET | XDP_TCP_FREE; + tcp_table_remove(tcp_table_re_lookup(rl->conn, tcp_table), tcp_table); // also updates tcp_table->next_* + + *free_conns -= 1; + *free_inbuf -= buffer_alloc_size(rl->conn->inbuf.iov_len); + *free_outbuf -= knot_tcp_outbufs_usage(rl->conn->outbufs); + + knot_sweep_stats_incr(stats, counter); +} + +_public_ +int knot_tcp_sweep(knot_tcp_table_t *tcp_table, + uint32_t close_timeout, uint32_t reset_timeout, + uint32_t resend_timeout, uint32_t limit_conn_count, + size_t limit_ibuf_size, size_t limit_obuf_size, + knot_tcp_relay_t *relays, uint32_t max_relays, + struct knot_sweep_stats *stats) +{ + if (tcp_table == NULL || relays == NULL || max_relays < 1) { + return KNOT_EINVAL; + } + + uint32_t now = get_timestamp(); + memset(relays, 0, max_relays * sizeof(*relays)); + knot_tcp_relay_t *rl = relays, *rl_max = rl + max_relays; + + ssize_t free_conns = (ssize_t)(tcp_table->usage - limit_conn_count); + ssize_t free_inbuf = (ssize_t)(tcp_table->inbufs_total - MIN(limit_ibuf_size, SSIZE_MAX)); + ssize_t free_outbuf = (ssize_t)(tcp_table->outbufs_total - MIN(limit_obuf_size, SSIZE_MAX)); + + // reset connections to free ibufs + while (free_inbuf > 0 && rl != rl_max) { + assert(tcp_table->next_ibuf != NULL); + if (tcp_table->next_ibuf->inbuf.iov_len == 0) { // this conn might have get rid of ibuf in the meantime + next_ptr_ibuf(&tcp_table->next_ibuf); + } + assert(tcp_table->next_ibuf != NULL); + rl->conn = tcp_table->next_ibuf; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, KNOT_SWEEP_CTR_LIMIT_IBUF); + rl++; + } + + // reset connections to free obufs + while (free_outbuf > 0 && rl != rl_max) { + assert(tcp_table->next_obuf != NULL); + if (knot_tcp_outbufs_usage(tcp_table->next_obuf->outbufs) == 0) { + next_ptr_obuf(&tcp_table->next_obuf); + } + assert(tcp_table->next_obuf != NULL); + rl->conn = tcp_table->next_obuf; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, KNOT_SWEEP_CTR_LIMIT_OBUF); + rl++; + } + + // reset connections to free their count, and old ones + knot_tcp_conn_t *conn, *next; + WALK_LIST_DELSAFE(conn, next, *tcp_table_timeout(tcp_table)) { + bool active = now - conn->last_active < reset_timeout; + if ((free_conns <= 0 && active) || rl == rl_max) { + break; + } + + knot_sweep_counter_t ctr = active ? KNOT_SWEEP_CTR_LIMIT_CONN : + KNOT_SWEEP_CTR_TIMEOUT_RST; + rl->conn = conn; + sweep_reset(tcp_table, rl, &free_conns, &free_inbuf, &free_outbuf, + stats, ctr); + rl++; + } + + // close old connections + while (tcp_table->next_close != NULL && + now - tcp_table->next_close->last_active >= close_timeout && + rl != rl_max) { + if (tcp_table->next_close->state != XDP_TCP_CLOSING1) { + rl->conn = tcp_table->next_close; + rl->answer = XDP_TCP_CLOSE; + knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_TIMEOUT); + rl++; + } + next_node_ptr(&tcp_table->next_close); + } + + // resend unACKed data + while (tcp_table->next_resend != NULL && + now - tcp_table->next_resend->last_active >= resend_timeout && + rl != rl_max) { + rl->conn = tcp_table->next_resend; + rl->answer = XDP_TCP_RESEND; + rl++; + next_ptr_obuf(&tcp_table->next_resend); + } + + return KNOT_EOK; +} + +_public_ +void knot_tcp_cleanup(knot_tcp_table_t *tcp_table, knot_tcp_relay_t relays[], + uint32_t relay_count) +{ + (void)tcp_table; + for (uint32_t i = 0; i < relay_count; i++) { + if (relays[i].answer & XDP_TCP_FREE) { + assert(conn_removed(relays[i].conn)); + assert(relays[i].conn != tcp_table->next_close); + assert(relays[i].conn != tcp_table->next_ibuf); + assert(relays[i].conn != tcp_table->next_obuf); + assert(relays[i].conn != tcp_table->next_resend); + + del_conn(relays[i].conn); + } + free(relays[i].inbf); + } + memset(relays, 0, relay_count * sizeof(relays[0])); +} diff --git a/src/libknot/xdp/tcp.h b/src/libknot/xdp/tcp.h new file mode 100644 index 0000000..09fe652 --- /dev/null +++ b/src/libknot/xdp/tcp.h @@ -0,0 +1,226 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief TCP over XDP IO interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include "libknot/xdp/msg.h" +#include "libknot/xdp/xdp.h" + +struct knot_sweep_stats; + +typedef enum { + XDP_TCP_NOOP = 0, + XDP_TCP_SYN = 1, + XDP_TCP_ESTABLISH = 2, + XDP_TCP_CLOSE = 3, + XDP_TCP_RESET = 4, + XDP_TCP_RESEND = 5, + + XDP_TCP_FREE = 0x10, +} knot_tcp_action_t; + +typedef enum { + XDP_TCP_NORMAL, + XDP_TCP_ESTABLISHING, + XDP_TCP_CLOSING1, // FIN+ACK sent + XDP_TCP_CLOSING2, // FIN+ACK received and sent +} knot_tcp_state_t; + +typedef enum { + XDP_TCP_FREE_NONE, + XDP_TCP_FREE_DATA, + XDP_TCP_FREE_PREFIX, +} knot_tcp_relay_free_t; + +typedef enum { + XDP_TCP_IGNORE_NONE = 0, + XDP_TCP_IGNORE_ESTABLISH = (1 << 0), + XDP_TCP_IGNORE_DATA_ACK = (1 << 1), + XDP_TCP_IGNORE_FIN = (1 << 2), +} knot_tcp_ignore_t; + +typedef struct knot_tcp_conn { + struct { + struct knot_tcp_conn *list_node_next; + struct knot_tcp_conn *list_node_prev; + } list_node_placeholder; + struct sockaddr_in6 ip_rem; + struct sockaddr_in6 ip_loc; + uint8_t last_eth_rem[ETH_ALEN]; + uint8_t last_eth_loc[ETH_ALEN]; + uint16_t mss; + uint8_t window_scale; + uint32_t seqno; + uint32_t ackno; + uint32_t acked; + uint32_t window_size; + uint32_t last_active; + uint32_t establish_rtt; // in microseconds + knot_tcp_state_t state; + struct iovec inbuf; + struct knot_tcp_outbuf *outbufs; + struct knot_tcp_conn *next; +} knot_tcp_conn_t; + +typedef struct { + size_t size; + size_t usage; + size_t inbufs_total; + size_t outbufs_total; + uint64_t hash_secret[2]; + knot_tcp_conn_t *next_close; + knot_tcp_conn_t *next_ibuf; + knot_tcp_conn_t *next_obuf; + knot_tcp_conn_t *next_resend; + knot_tcp_conn_t *conns[]; +} knot_tcp_table_t; + +typedef struct { + const knot_xdp_msg_t *msg; + knot_tcp_action_t action; + knot_xdp_msg_flag_t auto_answer; + uint32_t auto_seqno; + knot_tcp_action_t answer; + struct knot_tcp_inbufs_upd_res *inbf; + knot_tcp_conn_t *conn; +} knot_tcp_relay_t; + +/*! + * \brief Return next TCP sequence number. + */ +inline static uint32_t knot_tcp_next_seqno(const knot_xdp_msg_t *msg) +{ + uint32_t res = msg->seqno + msg->payload.iov_len; + if (msg->flags & (KNOT_XDP_MSG_SYN | KNOT_XDP_MSG_FIN)) { + res++; + } + return res; +} + +/*! + * \brief Check if the relay is empty. + */ +inline static bool knot_tcp_relay_empty(const knot_tcp_relay_t *relay) +{ + return relay->action == XDP_TCP_NOOP && relay->answer == XDP_TCP_NOOP && + relay->auto_answer == 0 && relay->inbf == NULL; +} + +/*! + * \brief Allocate TCP connection-handling hash table. + * + * \param size Number of records for the hash table. + * \param secret_share Optional: share the hashing secret with another table. + * + * \note Hashing conflicts are solved by single-linked-lists in each record. + * + * \return The table, or NULL. + */ +knot_tcp_table_t *knot_tcp_table_new(size_t size, knot_tcp_table_t *secret_share); + +/*! + * \brief Free TCP connection hash table including all connection records. + * + * \note The freed connections are not closed nor reset. + */ +void knot_tcp_table_free(knot_tcp_table_t *table); + +/*! + * \brief Process received packets, prepare automatic responses (e.g. ACK), pick incoming data. + * + * \param relays Out: relays to be filled with message/connection details. + * \param msgs Packets received by knot_xdp_recv(). + * \param msg_count Number of received packets. + * \param tcp_table Table of TCP connections. + * \param syn_table Optional: extra table for handling partially established connections. + * \param ignore Ignore specific TCP packets indication. + * + * \return KNOT_E* + */ +int knot_tcp_recv(knot_tcp_relay_t *relays, knot_xdp_msg_t msgs[], uint32_t msg_count, + knot_tcp_table_t *tcp_table, knot_tcp_table_t *syn_table, + knot_tcp_ignore_t ignore); + +/*! + * \brief Prepare data (payload) to be sent as a response on specific relay. + * + * \param relay Relay with active connection. + * \param tcp_table TCP table. + * \param ignore_lastbyte Evil mode: drop last byte of the payload. + * \param data Data payload, possibly > MSS and > window. + * \param len Payload length, < 64k. + * + * \return KNOT_E* + */ +int knot_tcp_reply_data(knot_tcp_relay_t *relay, knot_tcp_table_t *tcp_table, + bool ignore_lastbyte, uint8_t *data, uint32_t len); + +/*! + * \brief Send TCP packets. + * + * \param socket XDP socket to send through. + * \param relays Connection changes and data. + * \param relay_count Number of connection changes and data. + * \param max_at_once Limit of packet batch sent by knot_xdp_send(). + * + * \return KNOT_E* + */ +int knot_tcp_send(knot_xdp_socket_t *socket, knot_tcp_relay_t relays[], + uint32_t relay_count, uint32_t max_at_once); + +/*! + * \brief Cleanup old TCP connections, perform timeout checks. + * + * \param tcp_table TCP connection table to clean up. + * \param close_timeout Gracefully close connections older than this (usecs). + * \param reset_timeout Reset connections older than this (usecs). + * \param resend_timeout Resend unAcked data older than this (usecs). + * \param limit_conn_count Limit of active connections in TCP table, reset if more. + * \param limit_ibuf_size Limit of memory usage by input buffers, reset if exceeded. + * \param limit_obuf_size Limit of memory usage by output buffers, reset if exceeded. + * \param relays Out: relays to be filled with close/reset instructions for knot_tcp_send(). + * \param max_relays Maximum relays to be used. + * \param stats Out: sweeped out connection statistics. + * + * \return KNOT_E* + */ +int knot_tcp_sweep(knot_tcp_table_t *tcp_table, + uint32_t close_timeout, uint32_t reset_timeout, + uint32_t resend_timeout, uint32_t limit_conn_count, + size_t limit_ibuf_size, size_t limit_obuf_size, + knot_tcp_relay_t *relays, uint32_t max_relays, + struct knot_sweep_stats *stats); + +/*! + * \brief Free resources of closed/reset connections. + * + * \param tcp_table TCP table with connections. + * \param relays Relays with closed/reset (or other, ignored) connections. + * \param relay_count Number of relays. + */ +void knot_tcp_cleanup(knot_tcp_table_t *tcp_table, knot_tcp_relay_t relays[], + uint32_t relay_count); + +/*! @} */ diff --git a/src/libknot/xdp/tcp_iobuf.c b/src/libknot/xdp/tcp_iobuf.c new file mode 100644 index 0000000..3979d7c --- /dev/null +++ b/src/libknot/xdp/tcp_iobuf.c @@ -0,0 +1,302 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#include "libknot/xdp/tcp_iobuf.h" + +#include "contrib/macros.h" +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/error.h" +#include "libknot/wire.h" + +static void iov_clear(struct iovec *iov) +{ + free(iov->iov_base); + memset(iov, 0, sizeof(*iov)); +} + +static void iov_inc(struct iovec *iov, size_t shift) +{ + assert(shift <= iov->iov_len); + iov->iov_base += shift; + iov->iov_len -= shift; +} + +static size_t tcp_payload_len(const struct iovec *payload) +{ + if (payload->iov_len < 2) { + return 0; + } + return knot_wire_read_u16(payload->iov_base); +} + +static void iov_append(struct iovec *what, const struct iovec *with) +{ + // NOTE: what->iov_base must be pre-allocated large enough + memcpy(what->iov_base + what->iov_len, with->iov_base, with->iov_len); + what->iov_len += with->iov_len; +} + +static knot_tcp_inbufs_upd_res_t *tinbufu_alloc(size_t inbuf_count, size_t first_inbuf) +{ + knot_tcp_inbufs_upd_res_t *res = malloc(sizeof(*res) + inbuf_count * sizeof(struct iovec) + first_inbuf); + if (res == NULL) { + return NULL; + } + + res->next = NULL; + res->n_inbufs = inbuf_count; + return res; +} + +uint64_t buffer_alloc_size(uint64_t buffer_len) +{ + if (buffer_len == 0) { + return 0; + } + buffer_len -= 1; + buffer_len |= 0x3f; // the result will be at least 64 + buffer_len |= (buffer_len >> 1); + buffer_len |= (buffer_len >> 2); + buffer_len |= (buffer_len >> 4); + buffer_len |= (buffer_len >> 8); + buffer_len |= (buffer_len >> 16); + buffer_len |= (buffer_len >> 32); + return buffer_len + 1; +} + +_public_ +int knot_tcp_inbufs_upd(struct iovec *buffer, struct iovec data, bool alloc_bufs, + knot_tcp_inbufs_upd_res_t **result, size_t *buffers_total) +{ + knot_tcp_inbufs_upd_res_t *out = NULL; + struct iovec *cur = NULL; + + if (data.iov_len <= 0) { + return KNOT_EOK; + } + + // Finalize size bytes in buffer + assert(buffer != NULL && result != NULL && buffers_total != NULL); + if (buffer->iov_len == 1) { + assert(buffer->iov_base != NULL); + ((uint8_t *)buffer->iov_base)[1] = ((uint8_t *)data.iov_base)[0]; + buffer->iov_len++; + iov_inc(&data, 1); + if (data.iov_len <= 0) { + return KNOT_EOK; + } + } + + // find the end of linked list if not already + while (*result != NULL) { + result = &(*result)->next; + } + + // Count space needed for finished segments + size_t iov_cnt = 0, iov_bytesize = 0, message_len = 0; + struct iovec data_use = data; + bool skip_cnt = false; + if (buffer->iov_len >= 2) { + message_len = tcp_payload_len(buffer); + size_t data_offset = message_len - (buffer->iov_len - sizeof(uint16_t)); + if (data_use.iov_len >= data_offset) { + ++iov_cnt; + iov_bytesize += message_len; + iov_inc(&data_use, data_offset); + } else { + skip_cnt = true; + } + } + if (!skip_cnt) { + if (alloc_bufs) { + while (data_use.iov_len >= 2 && + (message_len = tcp_payload_len(&data_use)) <= (data_use.iov_len - sizeof(uint16_t))) { + ++iov_cnt; + iov_bytesize += message_len; + iov_inc(&data_use, message_len + sizeof(uint16_t)); + } + } else { + while (data_use.iov_len >= 2 && + (message_len = tcp_payload_len(&data_use)) <= (data_use.iov_len - sizeof(uint16_t))) { + ++iov_cnt; + iov_inc(&data_use, message_len + sizeof(uint16_t)); + } + } + } + + // Alloc linked-list node and copy data from `buffer` to output + if (iov_cnt > 0) { + out = tinbufu_alloc(iov_cnt, iov_bytesize); + if (out == NULL) { + return KNOT_ENOMEM; + } + + cur = out->inbufs; + uint8_t *out_buf_ptr = (uint8_t *)(cur + iov_cnt); + data_use = data; + if (buffer->iov_len >= 2) { // at least some data in buffer + struct iovec bf = { + .iov_base = buffer->iov_base + sizeof(uint16_t), + .iov_len = buffer->iov_len - sizeof(uint16_t) + }; + cur->iov_base = out_buf_ptr; + cur->iov_len = 0; + data_use.iov_base = data.iov_base; + data_use.iov_len = tcp_payload_len(buffer) - bf.iov_len; + iov_append(cur, &bf); + iov_append(cur, &data_use); + iov_inc(&data, data_use.iov_len); + out_buf_ptr = cur->iov_base + cur->iov_len; + ++cur; + *buffers_total -= buffer_alloc_size(buffer->iov_len); + iov_clear(buffer); + } + + if (alloc_bufs) { + for (; cur != out->inbufs + iov_cnt; ++cur) { + cur->iov_base = out_buf_ptr; + cur->iov_len = 0; + data_use.iov_len = tcp_payload_len(&data); + iov_inc(&data, 2); + data_use.iov_base = data.iov_base; + iov_append(cur, &data_use); + iov_inc(&data, data_use.iov_len); + out_buf_ptr = cur->iov_base + cur->iov_len; + } + } else { + for (; cur != out->inbufs + iov_cnt; ++cur) { + cur->iov_len = tcp_payload_len(&data); + iov_inc(&data, 2); + cur->iov_base = data.iov_base; + iov_inc(&data, cur->iov_len); + } + } + } + + // store the final incomplete payload to buffer + if (data.iov_len > 0) { + size_t buffer_original_size = buffer_alloc_size(buffer->iov_len); + size_t bufalloc = buffer_alloc_size(buffer->iov_len + data.iov_len); + if (buffer_original_size < bufalloc) { + void *newbuf = realloc(buffer->iov_base, bufalloc); + if (newbuf == NULL) { + free(buffer->iov_base); + buffer->iov_base = NULL; + free(out); + return KNOT_ENOMEM; + } + buffer->iov_base = newbuf; + *buffers_total += bufalloc - buffer_original_size; + } + iov_append(buffer, &data); + } + + *result = out; + + return KNOT_EOK; +} + +_public_ +int knot_tcp_outbufs_add(knot_tcp_outbuf_t **bufs, uint8_t *data, size_t len, + bool ignore_lastbyte, uint32_t mss, size_t *outbufs_total) +{ + if (len > UINT16_MAX) { + return KNOT_ELIMIT; + } + knot_tcp_outbuf_t **end = bufs; + while (*end != NULL) { // NOTE: this can be optimized by adding "end" pointer for the price of larger knot_tcp_conn_t struct + end = &(*end)->next; + } + uint16_t prefix = htobe16(len), prefix_len = sizeof(prefix); + while (len > 0) { + uint16_t newlen = MIN(len + prefix_len, mss); + knot_tcp_outbuf_t *newob = calloc(1, sizeof(*newob) + newlen); + if (newob == NULL) { + return KNOT_ENOMEM; + } + *outbufs_total += sizeof(*newob) + newlen; + newob->len = newlen; + if (ignore_lastbyte) { + newob->len--; + } + memcpy(newob->bytes, &prefix, prefix_len); + memcpy(newob->bytes + prefix_len, data, newlen - prefix_len); + + *end = newob; + end = &newob->next; + + data += newlen - prefix_len; + len -= newlen - prefix_len; + + prefix_len = 0; + } + return KNOT_EOK; +} + +static bool seqno_lower(uint32_t seqno, uint32_t ackno, uint32_t ackno_min) +{ + if (ackno_min <= ackno) { + return (seqno >= ackno_min && seqno <= ackno); + } else { + return (seqno >= ackno_min || seqno <= ackno); + } +} + +_public_ +void knot_tcp_outbufs_ack(knot_tcp_outbuf_t **bufs, uint32_t ackno, size_t *outbufs_total) +{ + uint32_t ackno_min = ackno - (UINT32_MAX / 2); // FIXME better? + while (*bufs != NULL && (*bufs)->sent && seqno_lower((*bufs)->seqno + (*bufs)->len, ackno, ackno_min)) { + knot_tcp_outbuf_t *tofree = *bufs; + *bufs = tofree->next; + *outbufs_total -= tofree->len + sizeof(*tofree); + free(tofree); + } +} + +_public_ +void knot_tcp_outbufs_can_send(knot_tcp_outbuf_t *bufs, ssize_t window_size, bool resend, + knot_tcp_outbuf_t **send_start, size_t *send_count) +{ + *send_count = 0; + *send_start = bufs; + while (*send_start != NULL && (*send_start)->sent && !resend) { + window_size -= (*send_start)->len; + *send_start = (*send_start)->next; + } + + knot_tcp_outbuf_t *can_send = *send_start; + while (can_send != NULL && window_size >= can_send->len) { + (*send_count)++; + window_size -= can_send->len; + can_send = can_send->next; + } +} + +_public_ +size_t knot_tcp_outbufs_usage(knot_tcp_outbuf_t *bufs) +{ + size_t res = 0; + for (knot_tcp_outbuf_t *i = bufs; i != NULL; i = i->next) { + res += i->len + sizeof(*i); + } + return res; +} diff --git a/src/libknot/xdp/tcp_iobuf.h b/src/libknot/xdp/tcp_iobuf.h new file mode 100644 index 0000000..5a076e6 --- /dev/null +++ b/src/libknot/xdp/tcp_iobuf.h @@ -0,0 +1,129 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief TCP buffer helpers. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <sys/uio.h> + +typedef struct knot_tcp_outbuf { + struct knot_tcp_outbuf *next; + uint32_t len; + uint32_t seqno; + bool sent; + uint8_t bytes[]; +} knot_tcp_outbuf_t; + +typedef enum { + KNOT_SWEEP_CTR_TIMEOUT = 0, + KNOT_SWEEP_CTR_LIMIT_CONN = 1, + KNOT_SWEEP_CTR_LIMIT_IBUF = 2, + KNOT_SWEEP_CTR_LIMIT_OBUF = 3, + KNOT_SWEEP_CTR_TIMEOUT_RST = 4, +} knot_sweep_counter_t; + +typedef struct knot_sweep_stats { + uint64_t last_log; // in seconds + uint32_t total; + uint32_t counters[5]; +} knot_sweep_stats_t; + +typedef struct knot_tcp_inbufs_upd_res { + size_t n_inbufs; + struct knot_tcp_inbufs_upd_res *next; + struct iovec inbufs[]; +} knot_tcp_inbufs_upd_res_t; + +inline static void knot_sweep_stats_incr(knot_sweep_stats_t *stats, knot_sweep_counter_t counter) +{ + (stats->counters[counter])++; + (stats->total)++; +} + +inline static void knot_sweep_stats_reset(knot_sweep_stats_t *stats) +{ + memset(stats, 0, sizeof(*stats)); +} + +uint64_t buffer_alloc_size(uint64_t buffer_len); + +/*! + * \brief Handle DNS-over-TCP payloads in buffer and message. + * + * \param buffer In/out: persistent buffer to store incomplete DNS payloads between receiving packets. + * \param data In: momental DNS payloads in incoming packet. + * \param alloc_bufs In: allocate extra buffers and always copy data instead of pointing inside recvd data. + * \param result Out: list of incoming DNS messages. + * \param buffers_total In/Out: total size of buffers (will be increased or decreased). + * + * \return KNOT_EOK, KNOT_ENOMEM + */ +int knot_tcp_inbufs_upd(struct iovec *buffer, struct iovec data, bool alloc_bufs, + knot_tcp_inbufs_upd_res_t **result, size_t *buffers_total); + +/*! + * \brief Add payload to be sent by TCP, to output buffers. + * + * \param bufs Output buffers to be updated. + * \param data Payload to be sent. + * \param len Payload length. + * \param ignore_lastbyte Evil mode: drop last byte of the payload. + * \param mss Connection outgoing MSS. + * \param outbufs_total In/out: total outbuf statistic to be updated. + * + * \return KNOT_E* + */ +int knot_tcp_outbufs_add(knot_tcp_outbuf_t **bufs, uint8_t *data, size_t len, + bool ignore_lastbyte, uint32_t mss, size_t *outbufs_total); + +/*! + * \brief Remove+free acked data from output buffers. + * + * \param bufs Output buffers to be updated. + * \param ackno Ackno of received ACK. + * \param outbufs_total In/out: total outbuf statistic to be updated. + */ +void knot_tcp_outbufs_ack(knot_tcp_outbuf_t **bufs, uint32_t ackno, size_t *outbufs_total); + +/*! + * \brief Prepare output buffers to be sent now. + * + * \param bufs Output buffers to be updated. + * \param window_size Connection outgoing window size. + * \param resend Send also possibly already sent data. + * \param send_start Out: first output buffer to be sent. + * \param send_count Out: number of output buffers to be sent. + */ +void knot_tcp_outbufs_can_send(knot_tcp_outbuf_t *bufs, ssize_t window_size, bool resend, + knot_tcp_outbuf_t **send_start, size_t *send_count); + +/*! + * \brief Compute allocated size of output buffers. + */ +size_t knot_tcp_outbufs_usage(knot_tcp_outbuf_t *bufs); + +/*! @} */ diff --git a/src/libknot/xdp/xdp.c b/src/libknot/xdp/xdp.c new file mode 100644 index 0000000..8286884 --- /dev/null +++ b/src/libknot/xdp/xdp.c @@ -0,0 +1,585 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <errno.h> +#include <netinet/in.h> +#include <linux/if_ether.h> +#include <linux/udp.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include "libknot/attribute.h" +#include "libknot/endian.h" +#include "libknot/errcode.h" +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/bpf-user.h" +#include "libknot/xdp/eth.h" +#include "libknot/xdp/msg_init.h" +#include "libknot/xdp/protocols.h" +#include "libknot/xdp/xdp.h" +#include "contrib/macros.h" +#include "contrib/net.h" + +#define FRAME_SIZE 2048 + +#define FRAME_COUNT_TX 2048 +#define FRAME_COUNT_RX 2048 +#define FRAME_COUNT (FRAME_COUNT_TX + FRAME_COUNT_RX) + +#define RING_LEN_TX FRAME_COUNT_TX +#define RING_LEN_CQ FRAME_COUNT_TX +#define RING_LEN_RX FRAME_COUNT_RX +/* It's recommended that the FQ ring size >= HW RX ring size + AF_XDP RX ring size. */ +#define RING_LEN_FQ (2 * FRAME_COUNT_RX) + +#define ALLOC_RETRY_NUM 15 +#define ALLOC_RETRY_DELAY 20 // In nanoseconds. + +/* With recent compilers we statically check #defines for settings that + * get refused by AF_XDP drivers (in current versions, at least). */ +#if (__STDC_VERSION__ >= 201112L) +#define IS_POWER_OF_2(n) (((n) & (n - 1)) == 0) +_Static_assert((FRAME_SIZE == 4096 || FRAME_SIZE == 2048) + && IS_POWER_OF_2(RING_LEN_TX) && IS_POWER_OF_2(RING_LEN_RX) + && IS_POWER_OF_2(RING_LEN_CQ) && IS_POWER_OF_2(RING_LEN_FQ) + && FRAME_COUNT_TX <= (1 << 16) /* see tx_free_indices */ + , "Incorrect #define combination for AF_XDP."); +#endif + +struct umem_frame { + uint8_t bytes[FRAME_SIZE]; +}; + +static int configure_xsk_umem(struct kxsk_umem **out_umem, bool extra_frames) +{ + /* Allocate memory and call driver to create the UMEM. */ + struct kxsk_umem *umem = calloc(1, + offsetof(struct kxsk_umem, tx_free_indices) + + sizeof(umem->tx_free_indices[0]) * FRAME_COUNT_TX); + if (umem == NULL) { + return KNOT_ENOMEM; + } + + size_t frame_count = FRAME_COUNT + (extra_frames ? FRAME_COUNT_RX : 0); + + int ret = posix_memalign((void **)&umem->frames, getpagesize(), + FRAME_SIZE * frame_count); + if (ret != 0) { + free(umem); + return KNOT_ENOMEM; + } + + const struct xsk_umem_config config = { + .fill_size = RING_LEN_FQ, + .comp_size = RING_LEN_CQ, + .frame_size = FRAME_SIZE, + .frame_headroom = KNOT_XDP_PKT_ALIGNMENT, + }; + + ret = xsk_umem__create(&umem->umem, umem->frames, FRAME_SIZE * frame_count, + &umem->fq, &umem->cq, &config); + if (ret != KNOT_EOK) { + free(umem->frames); + free(umem); + return ret; + } + *out_umem = umem; + + /* Designate the starting chunk of buffers for TX, and put them onto the stack. */ + umem->tx_free_count = FRAME_COUNT_TX; + for (uint32_t i = 0; i < FRAME_COUNT_TX; ++i) { + umem->tx_free_indices[i] = i; + } + + /* Designate the rest of buffers for RX, and pass them to the driver. */ + uint32_t idx = 0; + ret = xsk_ring_prod__reserve(&umem->fq, frame_count - FRAME_COUNT_TX, &idx); + if (ret != frame_count - FRAME_COUNT_TX) { + assert(0); + return KNOT_ERROR; + } + assert(idx == 0); + for (uint32_t i = FRAME_COUNT_TX; i < frame_count; ++i) { + *xsk_ring_prod__fill_addr(&umem->fq, idx++) = i * FRAME_SIZE; + } + xsk_ring_prod__submit(&umem->fq, frame_count - FRAME_COUNT_TX); + + return KNOT_EOK; +} + +static void deconfigure_xsk_umem(struct kxsk_umem *umem) +{ + (void)xsk_umem__delete(umem->umem); + free(umem->frames); + free(umem); +} + +static int configure_xsk_socket(struct kxsk_umem *umem, + const struct kxsk_iface *iface, + knot_xdp_socket_t **out_sock, + const knot_xdp_config_t *config) +{ + knot_xdp_socket_t *xsk_info = calloc(1, sizeof(*xsk_info)); + if (xsk_info == NULL) { + return KNOT_ENOMEM; + } + xsk_info->iface = iface; + xsk_info->umem = umem; + + uint16_t bind_flags = 0; + if (config != NULL && config->force_copy) { + bind_flags |= XDP_COPY; + } + + const struct xsk_socket_config sock_conf = { + .tx_size = RING_LEN_TX, + .rx_size = RING_LEN_RX, + .libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD, + .bind_flags = bind_flags, + }; + + int ret = xsk_socket__create(&xsk_info->xsk, iface->if_name, + iface->if_queue, umem->umem, + &xsk_info->rx, &xsk_info->tx, &sock_conf); + if (ret != 0) { + free(xsk_info); + return ret; + } + + *out_sock = xsk_info; + return KNOT_EOK; +} + +_public_ +int knot_xdp_init(knot_xdp_socket_t **socket, const char *if_name, int if_queue, + knot_xdp_filter_flag_t flags, uint16_t udp_port, uint16_t quic_port, + knot_xdp_load_bpf_t load_bpf, const knot_xdp_config_t *xdp_config) +{ + if (socket == NULL || if_name == NULL || + (udp_port == quic_port && (flags & KNOT_XDP_FILTER_UDP) && (flags & KNOT_XDP_FILTER_QUIC)) || + (flags & (KNOT_XDP_FILTER_UDP | KNOT_XDP_FILTER_TCP | KNOT_XDP_FILTER_QUIC)) == 0) { + return KNOT_EINVAL; + } + + struct kxsk_iface *iface; + const bool generic_xdp = (xdp_config != NULL && xdp_config->force_generic); + int ret = kxsk_iface_new(if_name, if_queue, load_bpf, generic_xdp, &iface); + if (ret != KNOT_EOK) { + return ret; + } + + /* Initialize shared packet_buffer for umem usage. */ + struct kxsk_umem *umem = NULL; + ret = configure_xsk_umem(&umem, xdp_config->extra_frames); + if (ret != KNOT_EOK) { + kxsk_iface_free(iface); + return ret; + } + + ret = configure_xsk_socket(umem, iface, socket, xdp_config); + if (ret != KNOT_EOK) { + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + return ret; + } + + (*socket)->frame_limit = FRAME_SIZE; + ret = knot_eth_mtu(if_name); + if (ret > 0) { + (*socket)->frame_limit = MIN((unsigned)ret, (*socket)->frame_limit); + } + + if (flags & KNOT_XDP_FILTER_ROUTE) { + ret = knot_eth_vlans(&(*socket)->vlan_map, &(*socket)->vlan_map_max); + if (ret != KNOT_EOK) { + xsk_socket__delete((*socket)->xsk); + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + free(*socket); + *socket = NULL; + return ret; + } + } + + ret = kxsk_socket_start(iface, flags, udp_port, quic_port, (*socket)->xsk); + if (ret != KNOT_EOK) { + free((*socket)->vlan_map); + xsk_socket__delete((*socket)->xsk); + deconfigure_xsk_umem(umem); + kxsk_iface_free(iface); + free(*socket); + *socket = NULL; + return ret; + } + + return ret; +} + +_public_ +void knot_xdp_deinit(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return; + } + if (unlikely(socket->send_mock != NULL)) { + free(socket); + return; + } + + kxsk_socket_stop(socket->iface); + xsk_socket__delete(socket->xsk); + deconfigure_xsk_umem(socket->umem); + + kxsk_iface_free((struct kxsk_iface *)/*const-cast*/socket->iface); + free(socket->vlan_map); + free(socket); +} + +_public_ +int knot_xdp_socket_fd(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return 0; + } + + return xsk_socket__fd(socket->xsk); +} + +static void tx_free_relative(struct kxsk_umem *umem, uint64_t addr_relative) +{ + /* The address may not point to *start* of buffer, but `/` solves that. */ + uint64_t index = addr_relative / FRAME_SIZE; + assert(index < FRAME_COUNT); + umem->tx_free_indices[umem->tx_free_count++] = index; +} + +_public_ +void knot_xdp_send_prepare(knot_xdp_socket_t *socket) +{ + if (socket == NULL || unlikely(socket->send_mock != NULL)) { + return; + } + + struct kxsk_umem *const umem = socket->umem; + struct xsk_ring_cons *const cq = &umem->cq; + + uint32_t idx = 0; + const uint32_t completed = xsk_ring_cons__peek(cq, UINT32_MAX, &idx); + if (completed == 0) { + return; + } + assert(umem->tx_free_count + completed <= FRAME_COUNT_TX); + + for (uint32_t i = 0; i < completed; ++i) { + uint64_t addr_relative = *xsk_ring_cons__comp_addr(cq, idx++); + tx_free_relative(umem, addr_relative); + } + + xsk_ring_cons__release(cq, completed); +} + +static struct umem_frame *alloc_tx_frame(knot_xdp_socket_t *socket) +{ + if (unlikely(socket->send_mock != NULL)) { + return malloc(sizeof(struct umem_frame)); + } + + const struct timespec delay = { .tv_nsec = ALLOC_RETRY_DELAY }; + struct kxsk_umem *umem = socket->umem; + + for (int i = 0; unlikely(umem->tx_free_count == 0); i++) { + if (i == ALLOC_RETRY_NUM) { + return NULL; + } + nanosleep(&delay, NULL); + knot_xdp_send_prepare(socket); + } + + uint32_t index = umem->tx_free_indices[--umem->tx_free_count]; + return umem->frames + index; +} + +static void prepare_payload(knot_xdp_msg_t *msg, void *uframe) +{ + size_t hdr_len = prot_write_hdrs_len(msg); + msg->payload.iov_base = uframe + hdr_len + KNOT_XDP_PKT_ALIGNMENT; + msg->payload.iov_len = FRAME_SIZE - hdr_len - KNOT_XDP_PKT_ALIGNMENT; +} + +_public_ +int knot_xdp_send_alloc(knot_xdp_socket_t *socket, knot_xdp_msg_flag_t flags, + knot_xdp_msg_t *out) +{ + if (socket == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct umem_frame *uframe = alloc_tx_frame(socket); + if (uframe == NULL) { + return KNOT_ENOMEM; + } + + msg_init(out, flags); + prepare_payload(out, uframe); + + return KNOT_EOK; +} + +_public_ +int knot_xdp_reply_alloc(knot_xdp_socket_t *socket, const knot_xdp_msg_t *query, + knot_xdp_msg_t *out) +{ + if (socket == NULL || query == NULL || out == NULL) { + return KNOT_EINVAL; + } + + struct umem_frame *uframe = alloc_tx_frame(socket); + if (uframe == NULL) { + return KNOT_ENOMEM; + } + + msg_init_reply(out, query); + prepare_payload(out, uframe); + + return KNOT_EOK; +} + +static void free_unsent(knot_xdp_socket_t *socket, const knot_xdp_msg_t *msg) +{ + if (unlikely(socket->send_mock != NULL)) { + free(msg->payload.iov_base - prot_write_hdrs_len(msg) - KNOT_XDP_PKT_ALIGNMENT); + return; + } + uint64_t addr_relative = (uint8_t *)msg->payload.iov_base + - socket->umem->frames->bytes; + tx_free_relative(socket->umem, addr_relative); +} + +_public_ +int knot_xdp_send(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count, uint32_t *sent) +{ + if (socket == NULL || msgs == NULL || sent == NULL) { + return KNOT_EINVAL; + } + if (unlikely(socket->send_mock != NULL)) { + int ret = socket->send_mock(socket, msgs, count, sent); + for (uint32_t i = 0; i < count; ++i) { + free_unsent(socket, &msgs[i]); + } + return ret; + } + + /* Now we want to do something close to + * xsk_ring_prod__reserve(&socket->tx, count, *idx) + * but we don't know in advance if we utilize *whole* `count`, + * and the API doesn't allow "cancelling reservations". + * Therefore we handle `socket->tx.cached_prod` by hand. + */ + if (xsk_prod_nb_free(&socket->tx, count) < count) { + /* This situation was sometimes observed in the emulated XDP mode. */ + for (uint32_t i = 0; i < count; ++i) { + free_unsent(socket, &msgs[i]); + } + return KNOT_ENOBUFS; + } + uint32_t idx = socket->tx.cached_prod; + + for (uint32_t i = 0; i < count; ++i) { + const knot_xdp_msg_t *msg = &msgs[i]; + + if (empty_msg(msg)) { + free_unsent(socket, msg); + } else { + size_t hdr_len = prot_write_hdrs_len(msg); + size_t tot_len = hdr_len + msg->payload.iov_len; + uint8_t *msg_beg = msg->payload.iov_base - hdr_len; + uint16_t mss = MIN(socket->frame_limit - hdr_len, KNOT_TCP_MSS); + prot_write_eth(msg_beg, msg, msg_beg + tot_len, mss); + + *xsk_ring_prod__tx_desc(&socket->tx, idx++) = (struct xdp_desc) { + .addr = msg_beg - socket->umem->frames->bytes, + .len = tot_len, + }; + } + } + + *sent = idx - socket->tx.cached_prod; + assert(*sent <= count); + socket->tx.cached_prod = idx; + xsk_ring_prod__submit(&socket->tx, *sent); + socket->kernel_needs_wakeup = true; + + return KNOT_EOK; +} + +_public_ +void knot_xdp_send_free(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count) +{ + for (uint32_t i = 0; i < count; i++) { + free_unsent(socket, &msgs[i]); + } +} + +_public_ +int knot_xdp_send_finish(knot_xdp_socket_t *socket) +{ + if (socket == NULL) { + return KNOT_EINVAL; + } + + /* Trigger sending queued packets. */ + if (!socket->kernel_needs_wakeup) { + return KNOT_EOK; + } + + int ret = sendto(xsk_socket__fd(socket->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + const bool is_ok = (ret >= 0); + // List of "safe" errors taken from + // https://github.com/torvalds/linux/blame/master/samples/bpf/xdpsock_user.c + const bool is_again = !is_ok && (errno == ENOBUFS || errno == EAGAIN + || errno == EBUSY || errno == ENETDOWN); + // Some of the !is_ok cases are a little unclear - what to do about the syscall, + // including how caller of _sendmsg_finish() should react. + if (is_ok || !is_again) { + socket->kernel_needs_wakeup = false; + } + if (is_again) { + return KNOT_EAGAIN; + } else if (is_ok) { + return KNOT_EOK; + } else { + return -errno; + } + /* This syscall might be avoided with a newer kernel feature (>= 5.4): + https://www.kernel.org/doc/html/latest/networking/af_xdp.html#xdp-use-need-wakeup-bind-flag + Unfortunately it's not easy to continue supporting older kernels + when using this feature on newer ones. + */ +} + +_public_ +int knot_xdp_recv(knot_xdp_socket_t *socket, knot_xdp_msg_t msgs[], + uint32_t max_count, uint32_t *count, size_t *wire_size) +{ + if (socket == NULL || msgs == NULL || count == NULL) { + return KNOT_EINVAL; + } + + uint32_t idx = 0; + const uint32_t available = xsk_ring_cons__peek(&socket->rx, max_count, &idx); + if (available == 0) { + *count = 0; + return KNOT_EOK; + } + assert(available <= max_count); + + for (uint32_t i = 0; i < available; ++i) { + knot_xdp_msg_t *msg = &msgs[i]; + const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&socket->rx, idx++); + uint8_t *uframe_p = (uint8_t *)socket->umem->frames + desc->addr; + + void *payl_end; + void *payl_start = prot_read_eth(uframe_p, msg, &payl_end, + socket->vlan_map, socket->vlan_map_max); + + msg->payload.iov_base = payl_start; + msg->payload.iov_len = payl_end - payl_start; + msg->mss = MIN(msg->mss, FRAME_SIZE - (payl_start - (void *)uframe_p)); + + if (wire_size != NULL) { + (*wire_size) += desc->len; + } + } + + xsk_ring_cons__release(&socket->rx, available); + *count = available; + + return KNOT_EOK; +} + +static uint8_t *msg_uframe_ptr(const knot_xdp_msg_t *msg) +{ + return NULL + ((msg->payload.iov_base - NULL) & ~(FRAME_SIZE - 1)); +} + +_public_ +void knot_xdp_recv_finish(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count) +{ + if (socket == NULL || msgs == NULL) { + return; + } + + const struct timespec delay = { .tv_nsec = ALLOC_RETRY_DELAY }; + + struct kxsk_umem *const umem = socket->umem; + struct xsk_ring_prod *const fq = &umem->fq; + + uint32_t idx = 0; + uint32_t reserved = xsk_ring_prod__reserve(fq, count, &idx); + for (int i = 0; unlikely(reserved < count); i++) { + if (i == ALLOC_RETRY_NUM) { + return; + } + nanosleep(&delay, NULL); + reserved = xsk_ring_prod__reserve(fq, count, &idx); + } + + for (uint32_t i = 0; i < reserved; ++i) { + uint8_t *uframe_p = msg_uframe_ptr(&msgs[i]); + uint64_t offset = uframe_p - umem->frames->bytes; + *xsk_ring_prod__fill_addr(fq, idx++) = offset; + } + + xsk_ring_prod__submit(fq, reserved); +} + +_public_ +void knot_xdp_socket_info(const knot_xdp_socket_t *socket, FILE *file) +{ + if (socket == NULL || file == NULL) { + return; + } + + // The number of busy frames + #define RING_BUSY(ring) \ + ((*(ring)->producer - *(ring)->consumer) & (ring)->mask) + + #define RING_PRINFO(name, ring) \ + fprintf(file, "Ring %s: size %4d, busy %4d (prod %4d, cons %4d)\n", \ + name, (unsigned)(ring)->size, \ + (unsigned)RING_BUSY((ring)), \ + (unsigned)*(ring)->producer, (unsigned)*(ring)->consumer) + + const int rx_busyf = RING_BUSY(&socket->umem->fq) + RING_BUSY(&socket->rx); + fprintf(file, "\nLOST RX frames: %4d", (int)(FRAME_COUNT_RX - rx_busyf)); + + const int tx_busyf = RING_BUSY(&socket->umem->cq) + RING_BUSY(&socket->tx); + const int tx_freef = socket->umem->tx_free_count; + fprintf(file, "\nLOST TX frames: %4d\n", (int)(FRAME_COUNT_TX - tx_busyf - tx_freef)); + + RING_PRINFO("FQ", &socket->umem->fq); + RING_PRINFO("RX", &socket->rx); + RING_PRINFO("TX", &socket->tx); + RING_PRINFO("CQ", &socket->umem->cq); + fprintf(file, "TX free frames: %4d\n", tx_freef); +} diff --git a/src/libknot/xdp/xdp.h b/src/libknot/xdp/xdp.h new file mode 100644 index 0000000..6c8bb1e --- /dev/null +++ b/src/libknot/xdp/xdp.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2023 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/*! + * \file + * + * \brief XDP IO interface. + * + * \addtogroup xdp + * @{ + */ + +#pragma once + +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <netinet/in.h> + +#include "libknot/xdp/bpf-consts.h" +#include "libknot/xdp/msg.h" + +/*! + * \brief Styles of loading BPF program. + * + * \note In *all* the cases loading can only succeed if at the end + * a compatible BPF program is loaded on the interface. + */ +typedef enum { + KNOT_XDP_LOAD_BPF_NEVER, /*!< Do not load; error out if not loaded already. */ + KNOT_XDP_LOAD_BPF_ALWAYS, /*!< Always load a program (overwrite it). */ + KNOT_XDP_LOAD_BPF_ALWAYS_UNLOAD, /*!< KNOT_XDP_LOAD_BPF_ALWAYS + unload previous. */ + KNOT_XDP_LOAD_BPF_MAYBE, /*!< Try with present program or load if none. */ + /* Implementation caveat: when re-using program in _MAYBE case, we get a message: + * libbpf: Kernel error message: XDP program already attached */ +} knot_xdp_load_bpf_t; + +/*! \brief Context structure for one XDP socket. */ +typedef struct knot_xdp_socket knot_xdp_socket_t; + +/*! \brief Configuration of XDP socket. */ +struct knot_xdp_config { + bool force_generic; /*!< Use generic XDP mode (avoid driver/hadrware implementation). */ + bool force_copy; /*!< Force copying packet data between kernel and user-space (avoid zero-copy). */ + bool extra_frames; /*!< Extra FQ frames. */ +}; + +/*! \brief Configuration of XDP socket. */ +typedef struct knot_xdp_config knot_xdp_config_t; + +/*! + * \brief Initialize XDP socket. + * + * \param socket XDP socket. + * \param if_name Name of the net iface (e.g. eth0). + * \param if_queue Network card queue to be used (normally 1 socket per each queue). + * \param flags XDP filter configuration flags. + * \param udp_port UDP and/or TCP port to listen on if enabled via \a opts. + * \param quic_port QUIC/UDP port to listen on if enabled via \a opts. + * \param load_bpf Insert BPF program into packet processing. + * \param xdp_config Optional XDP socket configuration. + * + * \return KNOT_E* or -errno + */ +int knot_xdp_init(knot_xdp_socket_t **socket, const char *if_name, int if_queue, + knot_xdp_filter_flag_t flags, uint16_t udp_port, uint16_t quic_port, + knot_xdp_load_bpf_t load_bpf, const knot_xdp_config_t *xdp_config); + +/*! + * \brief De-init XDP socket. + * + * \param socket XDP socket. + */ +void knot_xdp_deinit(knot_xdp_socket_t *socket); + +/*! + * \brief Return a file descriptor to be polled on for incoming packets. + * + * \param socket XDP socket. + * + * \return KNOT_E* + */ +int knot_xdp_socket_fd(knot_xdp_socket_t *socket); + +/*! + * \brief Collect completed TX buffers, so they can be used by knot_xdp_send_alloc(). + * + * \param socket XDP socket. + */ +void knot_xdp_send_prepare(knot_xdp_socket_t *socket); + +/*! + * \brief Allocate one buffer for an outgoing packet. + * + * \param socket XDP socket. + * \param flags Flags for new message. + * \param out Out: the allocated packet buffer. + * + * \return KNOT_E* + */ +int knot_xdp_send_alloc(knot_xdp_socket_t *socket, knot_xdp_msg_flag_t flags, + knot_xdp_msg_t *out); + +/*! + * \brief Allocate one buffer for a reply packet. + * + * \param socket XDP socket. + * \param query The packet to be replied to. + * \param out Out: the allocated packet buffer. + * + * \return KNOT_E* + */ +int knot_xdp_reply_alloc(knot_xdp_socket_t *socket, const knot_xdp_msg_t *query, + knot_xdp_msg_t *out); + +/*! + * \brief Send multiple packets thru XDP. + * + * \note The packets all must have been allocated by knot_xdp_send_alloc()! + * \note Do not free the packet payloads afterwards. + * \note Packets with zero length will be skipped. + * + * \param socket XDP socket. + * \param msgs Packets to be sent. + * \param count Number of packets. + * \param sent Out: number of packet successfully sent. + * + * \return KNOT_E* + */ +int knot_xdp_send(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count, uint32_t *sent); + +/*! + * \brief Cleanup messages that have not been knot_xdp_send(). + * + * ...possibly due to some error. + * + * \param socket XDP socket. + * \param msgs Messages to be freed. + * \param count Number of messages. + */ +void knot_xdp_send_free(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count); + +/*! + * \brief Syscall to kernel to wake up the network card driver after knot_xdp_send(). + * + * \param socket XDP socket. + * + * \return KNOT_E* or -errno + */ +int knot_xdp_send_finish(knot_xdp_socket_t *socket); + +/*! + * \brief Receive multiple packets thru XDP. + * + * \param socket XDP socket. + * \param msgs Out: buffers to be filled in with incoming packets. + * \param max_count Limit for number of packets received at once. + * \param count Out: real number of received packets. + * \param wire_size Out: (optional) total wire size of received packets. + * + * \return KNOT_E* + */ +int knot_xdp_recv(knot_xdp_socket_t *socket, knot_xdp_msg_t msgs[], + uint32_t max_count, uint32_t *count, size_t *wire_size); + +/*! + * \brief Free buffers with received packets. + * + * \param socket XDP socket. + * \param msgs Buffers with received packets. + * \param count Number of received packets to free. + */ +void knot_xdp_recv_finish(knot_xdp_socket_t *socket, const knot_xdp_msg_t msgs[], + uint32_t count); + +/*! + * \brief Print some info about the XDP socket. + * + * \param socket XDP socket. + * \param file Output file. + */ +void knot_xdp_socket_info(const knot_xdp_socket_t *socket, FILE *file); + +/*! @} */ |