From f5f56e1a1c4d9e9496fcb9d81131066a964ccd23 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 14:15:43 +0200 Subject: Adding upstream version 2.4.1. Signed-off-by: Daniel Baumann --- src/hooks/dhcp/high_availability/tests/Makefile.am | 70 + src/hooks/dhcp/high_availability/tests/Makefile.in | 1217 +++ .../tests/command_creator_unittest.cc | 530 ++ .../tests/communication_state_unittest.cc | 1201 +++ .../high_availability/tests/ha_config_unittest.cc | 1879 +++++ .../high_availability/tests/ha_impl_unittest.cc | 814 ++ .../dhcp/high_availability/tests/ha_mt_unittest.cc | 561 ++ .../high_availability/tests/ha_service_unittest.cc | 8037 ++++++++++++++++++++ src/hooks/dhcp/high_availability/tests/ha_test.cc | 378 + src/hooks/dhcp/high_availability/tests/ha_test.h | 304 + .../tests/lease_update_backlog_unittest.cc | 91 + .../tests/query_filter_unittest.cc | 1029 +++ .../dhcp/high_availability/tests/run_unittests.cc | 19 + 13 files changed, 16130 insertions(+) create mode 100644 src/hooks/dhcp/high_availability/tests/Makefile.am create mode 100644 src/hooks/dhcp/high_availability/tests/Makefile.in create mode 100644 src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_test.cc create mode 100644 src/hooks/dhcp/high_availability/tests/ha_test.h create mode 100644 src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc create mode 100644 src/hooks/dhcp/high_availability/tests/run_unittests.cc (limited to 'src/hooks/dhcp/high_availability/tests') diff --git a/src/hooks/dhcp/high_availability/tests/Makefile.am b/src/hooks/dhcp/high_availability/tests/Makefile.am new file mode 100644 index 0000000..f99f8eb --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/Makefile.am @@ -0,0 +1,70 @@ +SUBDIRS = . + +AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib +AM_CPPFLAGS += -I$(top_builddir)/src/hooks/dhcp/high_availability -I$(top_srcdir)/src/hooks/dhcp/high_availability +AM_CPPFLAGS += $(BOOST_INCLUDES) $(CRYPTO_CFLAGS) $(CRYPTO_INCLUDES) +AM_CPPFLAGS += -DLIBDHCP_HA_SO=\"$(abs_top_builddir)/src/hooks/dhcp/high_availability/.libs/libdhcp_ha.so\" +AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\" +TEST_CA_DIR = $(abs_top_srcdir)/src/lib/asiolink/testutils/ca +AM_CPPFLAGS += -DTEST_CA_DIR=\"$(TEST_CA_DIR)\" +TEST_HTTP_DIR = $(abs_top_srcdir)/src/lib/http/tests/testdata +AM_CPPFLAGS += -DTEST_HTTP_DIR=\"$(TEST_HTTP_DIR)\" + +AM_CXXFLAGS = $(KEA_CXXFLAGS) + +if USE_STATIC_LINK +AM_LDFLAGS = -static +endif + +# Unit test data files need to get installed. +EXTRA_DIST = + +CLEANFILES = *.gcno *.gcda + +TESTS_ENVIRONMENT = $(LIBTOOL) --mode=execute $(VALGRIND_COMMAND) + +TESTS = +if HAVE_GTEST +TESTS += ha_unittests + +ha_unittests_SOURCES = command_creator_unittest.cc +ha_unittests_SOURCES += communication_state_unittest.cc +ha_unittests_SOURCES += ha_config_unittest.cc +ha_unittests_SOURCES += ha_impl_unittest.cc +ha_unittests_SOURCES += ha_service_unittest.cc +ha_unittests_SOURCES += ha_test.cc ha_test.h +ha_unittests_SOURCES += ha_mt_unittest.cc +ha_unittests_SOURCES += lease_update_backlog_unittest.cc +ha_unittests_SOURCES += query_filter_unittest.cc +ha_unittests_SOURCES += run_unittests.cc + +ha_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES) $(LOG4CPLUS_INCLUDES) + +ha_unittests_LDFLAGS = $(AM_LDFLAGS) $(CRYPTO_LDFLAGS) $(GTEST_LDFLAGS) + +ha_unittests_CXXFLAGS = $(AM_CXXFLAGS) + +ha_unittests_LDADD = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la +ha_unittests_LDADD += $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la +ha_unittests_LDADD += $(top_builddir)/src/lib/process/libkea-process.la +ha_unittests_LDADD += $(top_builddir)/src/lib/eval/libkea-eval.la +ha_unittests_LDADD += $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la +ha_unittests_LDADD += $(top_builddir)/src/lib/stats/libkea-stats.la +ha_unittests_LDADD += $(top_builddir)/src/lib/config/libkea-cfgclient.la +ha_unittests_LDADD += $(top_builddir)/src/lib/http/libkea-http.la +ha_unittests_LDADD += $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la +ha_unittests_LDADD += $(top_builddir)/src/lib/hooks/libkea-hooks.la +ha_unittests_LDADD += $(top_builddir)/src/lib/database/libkea-database.la +ha_unittests_LDADD += $(top_builddir)/src/lib/cc/libkea-cc.la +ha_unittests_LDADD += $(top_builddir)/src/lib/asiolink/libkea-asiolink.la +ha_unittests_LDADD += $(top_builddir)/src/lib/dns/libkea-dns++.la +ha_unittests_LDADD += $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la +ha_unittests_LDADD += $(top_builddir)/src/lib/log/libkea-log.la +ha_unittests_LDADD += $(top_builddir)/src/lib/util/libkea-util.la +ha_unittests_LDADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la +ha_unittests_LDADD += $(LOG4CPLUS_LIBS) +ha_unittests_LDADD += $(CRYPTO_LIBS) +ha_unittests_LDADD += $(BOOST_LIBS) +ha_unittests_LDADD += $(GTEST_LDADD) +endif +noinst_PROGRAMS = $(TESTS) diff --git a/src/hooks/dhcp/high_availability/tests/Makefile.in b/src/hooks/dhcp/high_availability/tests/Makefile.in new file mode 100644 index 0000000..a346891 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/Makefile.in @@ -0,0 +1,1217 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +TESTS = $(am__EXEEXT_1) +@HAVE_GTEST_TRUE@am__append_1 = ha_unittests +noinst_PROGRAMS = $(am__EXEEXT_2) +subdir = src/hooks/dhcp/high_availability/tests +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4macros/ax_boost_for_kea.m4 \ + $(top_srcdir)/m4macros/ax_cpp11.m4 \ + $(top_srcdir)/m4macros/ax_cpp20.m4 \ + $(top_srcdir)/m4macros/ax_crypto.m4 \ + $(top_srcdir)/m4macros/ax_find_library.m4 \ + $(top_srcdir)/m4macros/ax_gssapi.m4 \ + $(top_srcdir)/m4macros/ax_gtest.m4 \ + $(top_srcdir)/m4macros/ax_isc_rpath.m4 \ + $(top_srcdir)/m4macros/ax_netconf.m4 \ + $(top_srcdir)/m4macros/libtool.m4 \ + $(top_srcdir)/m4macros/ltoptions.m4 \ + $(top_srcdir)/m4macros/ltsugar.m4 \ + $(top_srcdir)/m4macros/ltversion.m4 \ + $(top_srcdir)/m4macros/lt~obsolete.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +@HAVE_GTEST_TRUE@am__EXEEXT_1 = ha_unittests$(EXEEXT) +am__EXEEXT_2 = $(am__EXEEXT_1) +PROGRAMS = $(noinst_PROGRAMS) +am__ha_unittests_SOURCES_DIST = command_creator_unittest.cc \ + communication_state_unittest.cc ha_config_unittest.cc \ + ha_impl_unittest.cc ha_service_unittest.cc ha_test.cc \ + ha_test.h ha_mt_unittest.cc lease_update_backlog_unittest.cc \ + query_filter_unittest.cc run_unittests.cc +@HAVE_GTEST_TRUE@am_ha_unittests_OBJECTS = ha_unittests-command_creator_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-communication_state_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-ha_config_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-ha_impl_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-ha_service_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-ha_test.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-ha_mt_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-lease_update_backlog_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-query_filter_unittest.$(OBJEXT) \ +@HAVE_GTEST_TRUE@ ha_unittests-run_unittests.$(OBJEXT) +ha_unittests_OBJECTS = $(am_ha_unittests_OBJECTS) +am__DEPENDENCIES_1 = +@HAVE_GTEST_TRUE@ha_unittests_DEPENDENCIES = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/process/libkea-process.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/eval/libkea-eval.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/stats/libkea-stats.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/config/libkea-cfgclient.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/http/libkea-http.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/hooks/libkea-hooks.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/database/libkea-database.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cc/libkea-cc.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/asiolink/libkea-asiolink.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dns/libkea-dns++.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/log/libkea-log.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/util/libkea-util.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/exceptions/libkea-exceptions.la \ +@HAVE_GTEST_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ +@HAVE_GTEST_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +ha_unittests_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(ha_unittests_CXXFLAGS) \ + $(CXXFLAGS) $(ha_unittests_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = \ + ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po \ + ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po \ + ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po \ + ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po \ + ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po \ + ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po \ + ./$(DEPDIR)/ha_unittests-ha_test.Po \ + ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po \ + ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po \ + ./$(DEPDIR)/ha_unittests-run_unittests.Po +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_@AM_V@) +am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) +am__v_CXX_0 = @echo " CXX " $@; +am__v_CXX_1 = +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) +am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) +am__v_CXXLD_0 = @echo " CXXLD " $@; +am__v_CXXLD_1 = +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(ha_unittests_SOURCES) +DIST_SOURCES = $(am__ha_unittests_SOURCES_DIST) +RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \ + ctags-recursive dvi-recursive html-recursive info-recursive \ + install-data-recursive install-dvi-recursive \ + install-exec-recursive install-html-recursive \ + install-info-recursive install-pdf-recursive \ + install-ps-recursive install-recursive installcheck-recursive \ + installdirs-recursive pdf-recursive ps-recursive \ + tags-recursive uninstall-recursive +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +am__recursive_targets = \ + $(RECURSIVE_TARGETS) \ + $(RECURSIVE_CLEAN_TARGETS) \ + $(am__extra_recursive_targets) +AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \ + distdir distdir-am +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__tty_colors_dummy = \ + mgn= red= grn= lgn= blu= brg= std=; \ + am__color_tests=no +am__tty_colors = { \ + $(am__tty_colors_dummy); \ + if test "X$(AM_COLOR_TESTS)" = Xno; then \ + am__color_tests=no; \ + elif test "X$(AM_COLOR_TESTS)" = Xalways; then \ + am__color_tests=yes; \ + elif test "X$$TERM" != Xdumb && { test -t 1; } 2>/dev/null; then \ + am__color_tests=yes; \ + fi; \ + if test $$am__color_tests = yes; then \ + red=''; \ + grn=''; \ + lgn=''; \ + blu=''; \ + mgn=''; \ + brg=''; \ + std=''; \ + fi; \ +} +DIST_SUBDIRS = $(SUBDIRS) +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +ASCIIDOC = @ASCIIDOC@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BOOST_INCLUDES = @BOOST_INCLUDES@ +BOOST_LIBS = @BOOST_LIBS@ +BOTAN_TOOL = @BOTAN_TOOL@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CONTRIB_DIR = @CONTRIB_DIR@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPTO_CFLAGS = @CRYPTO_CFLAGS@ +CRYPTO_INCLUDES = @CRYPTO_INCLUDES@ +CRYPTO_LDFLAGS = @CRYPTO_LDFLAGS@ +CRYPTO_LIBS = @CRYPTO_LIBS@ +CRYPTO_PACKAGE = @CRYPTO_PACKAGE@ +CRYPTO_RPATH = @CRYPTO_RPATH@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DISTCHECK_BOOST_CONFIGURE_FLAG = @DISTCHECK_BOOST_CONFIGURE_FLAG@ +DISTCHECK_CONTRIB_CONFIGURE_FLAG = @DISTCHECK_CONTRIB_CONFIGURE_FLAG@ +DISTCHECK_CRYPTO_CONFIGURE_FLAG = @DISTCHECK_CRYPTO_CONFIGURE_FLAG@ +DISTCHECK_GSSAPI_CONFIGURE_FLAG = @DISTCHECK_GSSAPI_CONFIGURE_FLAG@ +DISTCHECK_GTEST_CONFIGURE_FLAG = @DISTCHECK_GTEST_CONFIGURE_FLAG@ +DISTCHECK_KEA_SHELL_CONFIGURE_FLAG = @DISTCHECK_KEA_SHELL_CONFIGURE_FLAG@ +DISTCHECK_LIBYANGCPP_CONFIGURE_FLAG = @DISTCHECK_LIBYANGCPP_CONFIGURE_FLAG@ +DISTCHECK_LIBYANG_CONFIGURE_FLAG = @DISTCHECK_LIBYANG_CONFIGURE_FLAG@ +DISTCHECK_LOG4CPLUS_CONFIGURE_FLAG = @DISTCHECK_LOG4CPLUS_CONFIGURE_FLAG@ +DISTCHECK_MYSQL_CONFIGURE_FLAG = @DISTCHECK_MYSQL_CONFIGURE_FLAG@ +DISTCHECK_PERFDHCP_CONFIGURE_FLAG = @DISTCHECK_PERFDHCP_CONFIGURE_FLAG@ +DISTCHECK_PGSQL_CONFIGURE_FLAG = @DISTCHECK_PGSQL_CONFIGURE_FLAG@ +DISTCHECK_PREMIUM_CONFIGURE_FLAG = @DISTCHECK_PREMIUM_CONFIGURE_FLAG@ +DISTCHECK_SYSREPOCPP_CONFIGURE_FLAG = @DISTCHECK_SYSREPOCPP_CONFIGURE_FLAG@ +DISTCHECK_SYSREPO_CONFIGURE_FLAG = @DISTCHECK_SYSREPO_CONFIGURE_FLAG@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GENHTML = @GENHTML@ +GREP = @GREP@ +GSSAPI_CFLAGS = @GSSAPI_CFLAGS@ +GSSAPI_LIBS = @GSSAPI_LIBS@ +GTEST_CONFIG = @GTEST_CONFIG@ +GTEST_INCLUDES = @GTEST_INCLUDES@ +GTEST_LDADD = @GTEST_LDADD@ +GTEST_LDFLAGS = @GTEST_LDFLAGS@ +GTEST_SOURCE = @GTEST_SOURCE@ +HAVE_NETCONF = @HAVE_NETCONF@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KEA_CXXFLAGS = @KEA_CXXFLAGS@ +KEA_SRCID = @KEA_SRCID@ +KRB5_CONFIG = @KRB5_CONFIG@ +LCOV = @LCOV@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LEX = @LEX@ +LEXLIB = @LEXLIB@ +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIBYANGCPP_CPPFLAGS = @LIBYANGCPP_CPPFLAGS@ +LIBYANGCPP_INCLUDEDIR = @LIBYANGCPP_INCLUDEDIR@ +LIBYANGCPP_LIBS = @LIBYANGCPP_LIBS@ +LIBYANGCPP_PREFIX = @LIBYANGCPP_PREFIX@ +LIBYANGCPP_VERSION = @LIBYANGCPP_VERSION@ +LIBYANG_CPPFLAGS = @LIBYANG_CPPFLAGS@ +LIBYANG_INCLUDEDIR = @LIBYANG_INCLUDEDIR@ +LIBYANG_LIBS = @LIBYANG_LIBS@ +LIBYANG_PREFIX = @LIBYANG_PREFIX@ +LIBYANG_VERSION = @LIBYANG_VERSION@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LOG4CPLUS_INCLUDES = @LOG4CPLUS_INCLUDES@ +LOG4CPLUS_LIBS = @LOG4CPLUS_LIBS@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MYSQL_CPPFLAGS = @MYSQL_CPPFLAGS@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PACKAGE_VERSION_TYPE = @PACKAGE_VERSION_TYPE@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PDFLATEX = @PDFLATEX@ +PERL = @PERL@ +PGSQL_CPPFLAGS = @PGSQL_CPPFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PKGPYTHONDIR = @PKGPYTHONDIR@ +PKG_CONFIG = @PKG_CONFIG@ +PLANTUML = @PLANTUML@ +PREMIUM_DIR = @PREMIUM_DIR@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +SED = @SED@ +SEP = @SEP@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SPHINXBUILD = @SPHINXBUILD@ +SRPD_PLUGINS_PATH = @SRPD_PLUGINS_PATH@ +SR_PLUGINS_PATH = @SR_PLUGINS_PATH@ +SR_REPO_PATH = @SR_REPO_PATH@ +STRIP = @STRIP@ +SYSREPOCPP_CPPFLAGS = @SYSREPOCPP_CPPFLAGS@ +SYSREPOCPP_INCLUDEDIR = @SYSREPOCPP_INCLUDEDIR@ +SYSREPOCPP_LIBS = @SYSREPOCPP_LIBS@ +SYSREPOCPP_PREFIX = @SYSREPOCPP_PREFIX@ +SYSREPOCPP_VERSION = @SYSREPOCPP_VERSION@ +SYSREPO_CPPFLAGS = @SYSREPO_CPPFLAGS@ +SYSREPO_INCLUDEDIR = @SYSREPO_INCLUDEDIR@ +SYSREPO_LIBS = @SYSREPO_LIBS@ +SYSREPO_PREFIX = @SYSREPO_PREFIX@ +SYSREPO_VERSION = @SYSREPO_VERSION@ +USE_LCOV = @USE_LCOV@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +WARNING_GCC_44_STRICT_ALIASING_CFLAG = @WARNING_GCC_44_STRICT_ALIASING_CFLAG@ +YACC = @YACC@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +SUBDIRS = . +AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib \ + -I$(top_builddir)/src/hooks/dhcp/high_availability \ + -I$(top_srcdir)/src/hooks/dhcp/high_availability \ + $(BOOST_INCLUDES) $(CRYPTO_CFLAGS) $(CRYPTO_INCLUDES) \ + -DLIBDHCP_HA_SO=\"$(abs_top_builddir)/src/hooks/dhcp/high_availability/.libs/libdhcp_ha.so\" \ + -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\" \ + -DTEST_CA_DIR=\"$(TEST_CA_DIR)\" \ + -DTEST_HTTP_DIR=\"$(TEST_HTTP_DIR)\" +TEST_CA_DIR = $(abs_top_srcdir)/src/lib/asiolink/testutils/ca +TEST_HTTP_DIR = $(abs_top_srcdir)/src/lib/http/tests/testdata +AM_CXXFLAGS = $(KEA_CXXFLAGS) +@USE_STATIC_LINK_TRUE@AM_LDFLAGS = -static + +# Unit test data files need to get installed. +EXTRA_DIST = +CLEANFILES = *.gcno *.gcda +TESTS_ENVIRONMENT = $(LIBTOOL) --mode=execute $(VALGRIND_COMMAND) +@HAVE_GTEST_TRUE@ha_unittests_SOURCES = command_creator_unittest.cc \ +@HAVE_GTEST_TRUE@ communication_state_unittest.cc \ +@HAVE_GTEST_TRUE@ ha_config_unittest.cc ha_impl_unittest.cc \ +@HAVE_GTEST_TRUE@ ha_service_unittest.cc ha_test.cc ha_test.h \ +@HAVE_GTEST_TRUE@ ha_mt_unittest.cc \ +@HAVE_GTEST_TRUE@ lease_update_backlog_unittest.cc \ +@HAVE_GTEST_TRUE@ query_filter_unittest.cc run_unittests.cc +@HAVE_GTEST_TRUE@ha_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES) $(LOG4CPLUS_INCLUDES) +@HAVE_GTEST_TRUE@ha_unittests_LDFLAGS = $(AM_LDFLAGS) $(CRYPTO_LDFLAGS) $(GTEST_LDFLAGS) +@HAVE_GTEST_TRUE@ha_unittests_CXXFLAGS = $(AM_CXXFLAGS) +@HAVE_GTEST_TRUE@ha_unittests_LDADD = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/process/libkea-process.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/eval/libkea-eval.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/stats/libkea-stats.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/config/libkea-cfgclient.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/http/libkea-http.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/hooks/libkea-hooks.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/database/libkea-database.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cc/libkea-cc.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/asiolink/libkea-asiolink.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dns/libkea-dns++.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/log/libkea-log.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/util/libkea-util.la \ +@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/exceptions/libkea-exceptions.la \ +@HAVE_GTEST_TRUE@ $(LOG4CPLUS_LIBS) $(CRYPTO_LIBS) \ +@HAVE_GTEST_TRUE@ $(BOOST_LIBS) $(GTEST_LDADD) +all: all-recursive + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/hooks/dhcp/high_availability/tests/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/hooks/dhcp/high_availability/tests/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +ha_unittests$(EXEEXT): $(ha_unittests_OBJECTS) $(ha_unittests_DEPENDENCIES) $(EXTRA_ha_unittests_DEPENDENCIES) + @rm -f ha_unittests$(EXEEXT) + $(AM_V_CXXLD)$(ha_unittests_LINK) $(ha_unittests_OBJECTS) $(ha_unittests_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-command_creator_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-communication_state_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_config_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_service_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_test.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-query_filter_unittest.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-run_unittests.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< + +ha_unittests-command_creator_unittest.o: command_creator_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-command_creator_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo -c -o ha_unittests-command_creator_unittest.o `test -f 'command_creator_unittest.cc' || echo '$(srcdir)/'`command_creator_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo $(DEPDIR)/ha_unittests-command_creator_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='command_creator_unittest.cc' object='ha_unittests-command_creator_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-command_creator_unittest.o `test -f 'command_creator_unittest.cc' || echo '$(srcdir)/'`command_creator_unittest.cc + +ha_unittests-command_creator_unittest.obj: command_creator_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-command_creator_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo -c -o ha_unittests-command_creator_unittest.obj `if test -f 'command_creator_unittest.cc'; then $(CYGPATH_W) 'command_creator_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/command_creator_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo $(DEPDIR)/ha_unittests-command_creator_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='command_creator_unittest.cc' object='ha_unittests-command_creator_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-command_creator_unittest.obj `if test -f 'command_creator_unittest.cc'; then $(CYGPATH_W) 'command_creator_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/command_creator_unittest.cc'; fi` + +ha_unittests-communication_state_unittest.o: communication_state_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-communication_state_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo -c -o ha_unittests-communication_state_unittest.o `test -f 'communication_state_unittest.cc' || echo '$(srcdir)/'`communication_state_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo $(DEPDIR)/ha_unittests-communication_state_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='communication_state_unittest.cc' object='ha_unittests-communication_state_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-communication_state_unittest.o `test -f 'communication_state_unittest.cc' || echo '$(srcdir)/'`communication_state_unittest.cc + +ha_unittests-communication_state_unittest.obj: communication_state_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-communication_state_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo -c -o ha_unittests-communication_state_unittest.obj `if test -f 'communication_state_unittest.cc'; then $(CYGPATH_W) 'communication_state_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/communication_state_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo $(DEPDIR)/ha_unittests-communication_state_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='communication_state_unittest.cc' object='ha_unittests-communication_state_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-communication_state_unittest.obj `if test -f 'communication_state_unittest.cc'; then $(CYGPATH_W) 'communication_state_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/communication_state_unittest.cc'; fi` + +ha_unittests-ha_config_unittest.o: ha_config_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_config_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo -c -o ha_unittests-ha_config_unittest.o `test -f 'ha_config_unittest.cc' || echo '$(srcdir)/'`ha_config_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo $(DEPDIR)/ha_unittests-ha_config_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_config_unittest.cc' object='ha_unittests-ha_config_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_config_unittest.o `test -f 'ha_config_unittest.cc' || echo '$(srcdir)/'`ha_config_unittest.cc + +ha_unittests-ha_config_unittest.obj: ha_config_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_config_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo -c -o ha_unittests-ha_config_unittest.obj `if test -f 'ha_config_unittest.cc'; then $(CYGPATH_W) 'ha_config_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_config_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo $(DEPDIR)/ha_unittests-ha_config_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_config_unittest.cc' object='ha_unittests-ha_config_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_config_unittest.obj `if test -f 'ha_config_unittest.cc'; then $(CYGPATH_W) 'ha_config_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_config_unittest.cc'; fi` + +ha_unittests-ha_impl_unittest.o: ha_impl_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_impl_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo -c -o ha_unittests-ha_impl_unittest.o `test -f 'ha_impl_unittest.cc' || echo '$(srcdir)/'`ha_impl_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo $(DEPDIR)/ha_unittests-ha_impl_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_impl_unittest.cc' object='ha_unittests-ha_impl_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_impl_unittest.o `test -f 'ha_impl_unittest.cc' || echo '$(srcdir)/'`ha_impl_unittest.cc + +ha_unittests-ha_impl_unittest.obj: ha_impl_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_impl_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo -c -o ha_unittests-ha_impl_unittest.obj `if test -f 'ha_impl_unittest.cc'; then $(CYGPATH_W) 'ha_impl_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_impl_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo $(DEPDIR)/ha_unittests-ha_impl_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_impl_unittest.cc' object='ha_unittests-ha_impl_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_impl_unittest.obj `if test -f 'ha_impl_unittest.cc'; then $(CYGPATH_W) 'ha_impl_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_impl_unittest.cc'; fi` + +ha_unittests-ha_service_unittest.o: ha_service_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_service_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo -c -o ha_unittests-ha_service_unittest.o `test -f 'ha_service_unittest.cc' || echo '$(srcdir)/'`ha_service_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo $(DEPDIR)/ha_unittests-ha_service_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_service_unittest.cc' object='ha_unittests-ha_service_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_service_unittest.o `test -f 'ha_service_unittest.cc' || echo '$(srcdir)/'`ha_service_unittest.cc + +ha_unittests-ha_service_unittest.obj: ha_service_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_service_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo -c -o ha_unittests-ha_service_unittest.obj `if test -f 'ha_service_unittest.cc'; then $(CYGPATH_W) 'ha_service_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_service_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo $(DEPDIR)/ha_unittests-ha_service_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_service_unittest.cc' object='ha_unittests-ha_service_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_service_unittest.obj `if test -f 'ha_service_unittest.cc'; then $(CYGPATH_W) 'ha_service_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_service_unittest.cc'; fi` + +ha_unittests-ha_test.o: ha_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_test.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_test.Tpo -c -o ha_unittests-ha_test.o `test -f 'ha_test.cc' || echo '$(srcdir)/'`ha_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_test.Tpo $(DEPDIR)/ha_unittests-ha_test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_test.cc' object='ha_unittests-ha_test.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_test.o `test -f 'ha_test.cc' || echo '$(srcdir)/'`ha_test.cc + +ha_unittests-ha_test.obj: ha_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_test.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_test.Tpo -c -o ha_unittests-ha_test.obj `if test -f 'ha_test.cc'; then $(CYGPATH_W) 'ha_test.cc'; else $(CYGPATH_W) '$(srcdir)/ha_test.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_test.Tpo $(DEPDIR)/ha_unittests-ha_test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_test.cc' object='ha_unittests-ha_test.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_test.obj `if test -f 'ha_test.cc'; then $(CYGPATH_W) 'ha_test.cc'; else $(CYGPATH_W) '$(srcdir)/ha_test.cc'; fi` + +ha_unittests-ha_mt_unittest.o: ha_mt_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_mt_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo -c -o ha_unittests-ha_mt_unittest.o `test -f 'ha_mt_unittest.cc' || echo '$(srcdir)/'`ha_mt_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo $(DEPDIR)/ha_unittests-ha_mt_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_mt_unittest.cc' object='ha_unittests-ha_mt_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_mt_unittest.o `test -f 'ha_mt_unittest.cc' || echo '$(srcdir)/'`ha_mt_unittest.cc + +ha_unittests-ha_mt_unittest.obj: ha_mt_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_mt_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo -c -o ha_unittests-ha_mt_unittest.obj `if test -f 'ha_mt_unittest.cc'; then $(CYGPATH_W) 'ha_mt_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_mt_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo $(DEPDIR)/ha_unittests-ha_mt_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_mt_unittest.cc' object='ha_unittests-ha_mt_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_mt_unittest.obj `if test -f 'ha_mt_unittest.cc'; then $(CYGPATH_W) 'ha_mt_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_mt_unittest.cc'; fi` + +ha_unittests-lease_update_backlog_unittest.o: lease_update_backlog_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-lease_update_backlog_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo -c -o ha_unittests-lease_update_backlog_unittest.o `test -f 'lease_update_backlog_unittest.cc' || echo '$(srcdir)/'`lease_update_backlog_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='lease_update_backlog_unittest.cc' object='ha_unittests-lease_update_backlog_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-lease_update_backlog_unittest.o `test -f 'lease_update_backlog_unittest.cc' || echo '$(srcdir)/'`lease_update_backlog_unittest.cc + +ha_unittests-lease_update_backlog_unittest.obj: lease_update_backlog_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-lease_update_backlog_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo -c -o ha_unittests-lease_update_backlog_unittest.obj `if test -f 'lease_update_backlog_unittest.cc'; then $(CYGPATH_W) 'lease_update_backlog_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/lease_update_backlog_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='lease_update_backlog_unittest.cc' object='ha_unittests-lease_update_backlog_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-lease_update_backlog_unittest.obj `if test -f 'lease_update_backlog_unittest.cc'; then $(CYGPATH_W) 'lease_update_backlog_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/lease_update_backlog_unittest.cc'; fi` + +ha_unittests-query_filter_unittest.o: query_filter_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-query_filter_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo -c -o ha_unittests-query_filter_unittest.o `test -f 'query_filter_unittest.cc' || echo '$(srcdir)/'`query_filter_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo $(DEPDIR)/ha_unittests-query_filter_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='query_filter_unittest.cc' object='ha_unittests-query_filter_unittest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-query_filter_unittest.o `test -f 'query_filter_unittest.cc' || echo '$(srcdir)/'`query_filter_unittest.cc + +ha_unittests-query_filter_unittest.obj: query_filter_unittest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-query_filter_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo -c -o ha_unittests-query_filter_unittest.obj `if test -f 'query_filter_unittest.cc'; then $(CYGPATH_W) 'query_filter_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/query_filter_unittest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo $(DEPDIR)/ha_unittests-query_filter_unittest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='query_filter_unittest.cc' object='ha_unittests-query_filter_unittest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-query_filter_unittest.obj `if test -f 'query_filter_unittest.cc'; then $(CYGPATH_W) 'query_filter_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/query_filter_unittest.cc'; fi` + +ha_unittests-run_unittests.o: run_unittests.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-run_unittests.o -MD -MP -MF $(DEPDIR)/ha_unittests-run_unittests.Tpo -c -o ha_unittests-run_unittests.o `test -f 'run_unittests.cc' || echo '$(srcdir)/'`run_unittests.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-run_unittests.Tpo $(DEPDIR)/ha_unittests-run_unittests.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='run_unittests.cc' object='ha_unittests-run_unittests.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-run_unittests.o `test -f 'run_unittests.cc' || echo '$(srcdir)/'`run_unittests.cc + +ha_unittests-run_unittests.obj: run_unittests.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-run_unittests.obj -MD -MP -MF $(DEPDIR)/ha_unittests-run_unittests.Tpo -c -o ha_unittests-run_unittests.obj `if test -f 'run_unittests.cc'; then $(CYGPATH_W) 'run_unittests.cc'; else $(CYGPATH_W) '$(srcdir)/run_unittests.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-run_unittests.Tpo $(DEPDIR)/ha_unittests-run_unittests.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='run_unittests.cc' object='ha_unittests-run_unittests.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-run_unittests.obj `if test -f 'run_unittests.cc'; then $(CYGPATH_W) 'run_unittests.cc'; else $(CYGPATH_W) '$(srcdir)/run_unittests.cc'; fi` + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +# This directory's subdirectories are mostly independent; you can cd +# into them and run 'make' without going through this Makefile. +# To change the values of 'make' variables: instead of editing Makefiles, +# (1) if the variable is set in 'config.status', edit 'config.status' +# (which will cause the Makefiles to be regenerated when you run 'make'); +# (2) otherwise, pass the desired values on the 'make' command line. +$(am__recursive_targets): + @fail=; \ + if $(am__make_keepgoing); then \ + failcom='fail=yes'; \ + else \ + failcom='exit 1'; \ + fi; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-recursive +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-recursive + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-recursive + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +check-TESTS: $(TESTS) + @failed=0; all=0; xfail=0; xpass=0; skip=0; \ + srcdir=$(srcdir); export srcdir; \ + list=' $(TESTS) '; \ + $(am__tty_colors); \ + if test -n "$$list"; then \ + for tst in $$list; do \ + if test -f ./$$tst; then dir=./; \ + elif test -f $$tst; then dir=; \ + else dir="$(srcdir)/"; fi; \ + if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xpass=`expr $$xpass + 1`; \ + failed=`expr $$failed + 1`; \ + col=$$red; res=XPASS; \ + ;; \ + *) \ + col=$$grn; res=PASS; \ + ;; \ + esac; \ + elif test $$? -ne 77; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xfail=`expr $$xfail + 1`; \ + col=$$lgn; res=XFAIL; \ + ;; \ + *) \ + failed=`expr $$failed + 1`; \ + col=$$red; res=FAIL; \ + ;; \ + esac; \ + else \ + skip=`expr $$skip + 1`; \ + col=$$blu; res=SKIP; \ + fi; \ + echo "$${col}$$res$${std}: $$tst"; \ + done; \ + if test "$$all" -eq 1; then \ + tests="test"; \ + All=""; \ + else \ + tests="tests"; \ + All="All "; \ + fi; \ + if test "$$failed" -eq 0; then \ + if test "$$xfail" -eq 0; then \ + banner="$$All$$all $$tests passed"; \ + else \ + if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \ + banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \ + fi; \ + else \ + if test "$$xpass" -eq 0; then \ + banner="$$failed of $$all $$tests failed"; \ + else \ + if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \ + banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \ + fi; \ + fi; \ + dashes="$$banner"; \ + skipped=""; \ + if test "$$skip" -ne 0; then \ + if test "$$skip" -eq 1; then \ + skipped="($$skip test was not run)"; \ + else \ + skipped="($$skip tests were not run)"; \ + fi; \ + test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$skipped"; \ + fi; \ + report=""; \ + if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \ + report="Please report to $(PACKAGE_BUGREPORT)"; \ + test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$report"; \ + fi; \ + dashes=`echo "$$dashes" | sed s/./=/g`; \ + if test "$$failed" -eq 0; then \ + col="$$grn"; \ + else \ + col="$$red"; \ + fi; \ + echo "$${col}$$dashes$${std}"; \ + echo "$${col}$$banner$${std}"; \ + test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \ + test -z "$$report" || echo "$${col}$$report$${std}"; \ + echo "$${col}$$dashes$${std}"; \ + test "$$failed" -eq 0; \ + else :; fi + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + $(am__make_dryrun) \ + || test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-TESTS +check: check-recursive +all-am: Makefile $(PROGRAMS) +installdirs: installdirs-recursive +installdirs-am: +install: install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-recursive + +clean-am: clean-generic clean-libtool clean-noinstPROGRAMS \ + mostlyclean-am + +distclean: distclean-recursive + -rm -f ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_test.Po + -rm -f ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-run_unittests.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -f ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-ha_test.Po + -rm -f ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po + -rm -f ./$(DEPDIR)/ha_unittests-run_unittests.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: + +.MAKE: $(am__recursive_targets) check-am install-am install-strip + +.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \ + am--depfiles check check-TESTS check-am clean clean-generic \ + clean-libtool clean-noinstPROGRAMS cscopelist-am ctags \ + ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs installdirs-am maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc new file mode 100644 index 0000000..8966b4c --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc @@ -0,0 +1,530 @@ +// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isc; +using namespace isc::asiolink; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::ha; + +namespace { + +/// @brief Creates lease instance used in tests. +/// +/// It creates a lease with an address of 192.1.2.3 and for the HW address +/// of 0b:0b:0b:0b:0b:0b. +/// +/// @return Pointer to the created lease. +Lease4Ptr createLease4() { + HWAddrPtr hwaddr(new HWAddr(std::vector(6, 11), HTYPE_ETHER)); + Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr, + static_cast(0), 0, + 60, 0, 1)); + return (lease4); +} + +/// @brief Creates IPv6 lease instance used in tests. +/// +/// It creates a lease with an address of 2001:db8:1::cafe and for the +/// DUID of 02:02:02:02:02:02:02:02. +/// +/// @return Pointer to the created lease. +Lease6Ptr createLease6() { + DuidPtr duid(new DUID(std::vector(8, 02))); + Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"), + duid, 1234, 50, 60, 1)); + return (lease6); +} + +/// @brief Returns JSON representation of the lease. +/// +/// @param lease_ptr Pointer to the lease. +/// @return Pointer to the JSON representation of the lease. +ElementPtr leaseAsJson(const LeasePtr& lease_ptr) { + ElementPtr lease = boost::const_pointer_cast(lease_ptr->toElement()); + // Replace cltt with expiration time as this is what the lease_cmds hooks + // library expects. + int64_t cltt = lease->get("cltt")->intValue(); + int64_t valid_lifetime = lease->get("valid-lft")->intValue(); + int64_t expire = cltt + valid_lifetime; + lease->set("expire", Element::create(expire)); + lease->remove("cltt"); + return (lease); +} + +/// @brief Performs basic checks on the command. +/// +/// It tests whether the command name is correct, if it contains arguments +/// and if the arguments are held in a map. +/// +/// @param command Pointer to the command to be tested. +/// @param expected_command Expected command name. +/// @param expected_service Expected service name. +/// @param [out] arguments Pointer to the arguments map extracted from the +/// provided command. +void +testCommandBasics(const ConstElementPtr& command, + const std::string& expected_command, + const std::string& expected_service, + ConstElementPtr& arguments) { + ASSERT_TRUE(command); + ASSERT_EQ(Element::map, command->getType()); + ConstElementPtr command_name = command->get("command"); + ASSERT_TRUE(command_name); + + // Make sure the command is a string. + ASSERT_EQ(Element::string, command_name->getType()); + // Verify the command name. + ASSERT_EQ(expected_command, command_name->stringValue()); + + // Make sure that the service is present and includes a single + // entry. + ConstElementPtr service = command->get("service"); + ASSERT_TRUE(service); + ASSERT_EQ(Element::list, service->getType()); + ASSERT_EQ(1, service->size()); + ASSERT_EQ(Element::string, service->get(0)->getType()); + EXPECT_EQ(expected_service, service->get(0)->stringValue()); + + // Make sure that arguments are present. + ConstElementPtr command_arguments = command->get("arguments"); + ASSERT_TRUE(command_arguments); + // Make sure that arguments are held in a map. + ASSERT_EQ(Element::map, command_arguments->getType()); + + // Return extracted arguments. + arguments = command_arguments; +} + +/// @brief Performs basic checks on the command. +/// +/// This variant of the function expects no arguments to be provided. +/// +/// @param command Pointer to the command to be tested. +/// @param expected_command Expected command name. +/// @param expected_service Expected service name. +void +testCommandBasics(const ConstElementPtr& command, + const std::string& expected_command, + const std::string& expected_service) { + ASSERT_TRUE(command); + ASSERT_EQ(Element::map, command->getType()); + ConstElementPtr command_name = command->get("command"); + ASSERT_TRUE(command_name); + + // Make sure the command is a string. + ASSERT_EQ(Element::string, command_name->getType()); + // Verify the command name. + ASSERT_EQ(expected_command, command_name->stringValue()); + + // Make sure that the service is present and includes a single + // entry. + ConstElementPtr service = command->get("service"); + ASSERT_TRUE(service); + ASSERT_EQ(Element::list, service->getType()); + ASSERT_EQ(1, service->size()); + ASSERT_EQ(Element::string, service->get(0)->getType()); + EXPECT_EQ(expected_service, service->get(0)->stringValue()); + + ConstElementPtr command_arguments = command->get("arguments"); + ASSERT_FALSE(command_arguments); +} + +// This test verifies that the dhcp-disable command is correct. +TEST(CommandCreatorTest, createDHCPDisable4) { + // Create command with max-period value set to 20. + ConstElementPtr command = CommandCreator::createDHCPDisable(20, HAServerType::DHCPv4); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp4", + arguments)); + ASSERT_EQ(2, arguments->size()); + ConstElementPtr max_period = arguments->get("max-period"); + ASSERT_TRUE(max_period); + ASSERT_EQ(Element::integer, max_period->getType()); + EXPECT_EQ(20, max_period->intValue()); + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); + + // Repeat the test but this time the max-period is not specified. + command = CommandCreator::createDHCPDisable(0, HAServerType::DHCPv4); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp4", + arguments)); + ASSERT_EQ(1, arguments->size()); + origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); +} + +// This test verifies that the dhcp-enable command is correct. +TEST(CommandCreatorTest, createDHCPEnable4) { + ConstElementPtr arguments; + ConstElementPtr command = CommandCreator::createDHCPEnable(HAServerType::DHCPv4); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-enable", "dhcp4", + arguments)); + ASSERT_EQ(1, arguments->size()); + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); +} + +// This test verifies that the ha-reset command sent to DHCPv4 server is correct. +TEST(CommandCreatorTest, createHAReset4) { + ConstElementPtr command = CommandCreator::createHAReset(HAServerType::DHCPv4); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-reset", "dhcp4")); +} + +// This test verifies that the ha-heartbeat command is correct. +TEST(CommandCreatorTest, createHeartbeat4) { + ConstElementPtr command = CommandCreator::createHeartbeat(HAServerType::DHCPv4); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-heartbeat", "dhcp4")); +} + +// This test verifies that the command generated for the lease update +// is correct. +TEST(CommandCreatorTest, createLease4Update) { + ConstElementPtr command = CommandCreator::createLease4Update(*createLease4()); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-update", "dhcp4", + arguments)); + ElementPtr lease_as_json = leaseAsJson(createLease4()); + // The lease update must contain the "force-create" parameter indicating that + // the lease must be created if it doesn't exist. + lease_as_json->set("force-create", Element::create(true)); + lease_as_json->set("origin", Element::create("ha-partner")); + EXPECT_EQ(lease_as_json->str(), arguments->str()); +} + +// This test verifies that the command generated for the lease deletion +// is correct. +TEST(CommandCreatorTest, createLease4Delete) { + ConstElementPtr command = CommandCreator::createLease4Delete(*createLease4()); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-del", "dhcp4", + arguments)); + ElementPtr lease_as_json = leaseAsJson(createLease4()); + lease_as_json->set("origin", Element::create("ha-partner")); + EXPECT_EQ(lease_as_json->str(), arguments->str()); +} + +// This test verifies that the lease4-get-all command is correct. +TEST(CommandCreatorTest, createLease4GetAll) { + ConstElementPtr command = CommandCreator::createLease4GetAll(); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-all", "dhcp4")); +} + +// This test verifies that the lease4-get-page command is correct when +// first page is fetched. +TEST(CommandCreatorTest, createLease4GetPageStart) { + Lease4Ptr lease4; + ConstElementPtr command = CommandCreator::createLease4GetPage(lease4, 10); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-page", "dhcp4", + arguments)); + + ConstElementPtr from = arguments->get("from"); + ASSERT_TRUE(from); + EXPECT_EQ(Element::string, from->getType()); + EXPECT_EQ("start", from->stringValue()); + + ConstElementPtr limit = arguments->get("limit"); + ASSERT_TRUE(limit); + ASSERT_EQ(Element::integer, limit->getType()); + EXPECT_EQ(10, limit->intValue()); +} + +// This test verifies that the lease4-get-page command is correct when next +// page is fetched. +TEST(CommandCreatorTest, createLease4GetPageAddress) { + Lease4Ptr lease4(new Lease4()); + lease4->addr_ = IOAddress("1.2.3.4"); + + ConstElementPtr command = CommandCreator::createLease4GetPage(lease4, 15); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-page", "dhcp4", + arguments)); + + ConstElementPtr from = arguments->get("from"); + ASSERT_TRUE(from); + EXPECT_EQ(Element::string, from->getType()); + EXPECT_EQ("1.2.3.4", from->stringValue()); + + ConstElementPtr limit = arguments->get("limit"); + ASSERT_TRUE(limit); + ASSERT_EQ(Element::integer, limit->getType()); + EXPECT_EQ(15, limit->intValue()); +} + +// This test verifies that exception is thrown if limit is set to 0 while +// creating lease4-get-page command. +TEST(CommandCreatorTest, createLease4GetPageZeroLimit) { + Lease4Ptr lease4; + EXPECT_THROW(CommandCreator::createLease4GetPage(lease4, 0), BadValue); +} + +// This test verifies that the dhcp-disable command (DHCPv6 case) is +// correct. +TEST(CommandCreatorTest, createDHCPDisable6) { + // Create command with max-period value set to 20. + ConstElementPtr command = CommandCreator::createDHCPDisable(20, HAServerType::DHCPv6); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp6", + arguments)); + ASSERT_EQ(2, arguments->size()); + ConstElementPtr max_period = arguments->get("max-period"); + ASSERT_TRUE(max_period); + ASSERT_EQ(Element::integer, max_period->getType()); + EXPECT_EQ(20, max_period->intValue()); + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); + + // Repeat the test but this time the max-period is not specified. + command = CommandCreator::createDHCPDisable(0, HAServerType::DHCPv6); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp6", + arguments)); + ASSERT_EQ(1, arguments->size()); + origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); +} + +// This test verifies that the dhcp-enable command (DHCPv6 case) is +// correct. +TEST(CommandCreatorTest, createDHCPEnable6) { + ConstElementPtr arguments; + ConstElementPtr command = CommandCreator::createDHCPEnable(HAServerType::DHCPv6); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-enable", "dhcp6", + arguments)); + ASSERT_EQ(1, arguments->size()); + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); +} + +// This test verifies that the ha-reset command sent to DHCPv6 server is correct. +TEST(CommandCreatorTest, createHAReset6) { + ConstElementPtr command = CommandCreator::createHAReset(HAServerType::DHCPv6); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-reset", "dhcp6")); +} + +// This test verifies that the command generated for the lease update +// is correct. +TEST(CommandCreatorTest, createLease6Update) { + ConstElementPtr command = CommandCreator::createLease6Update(*createLease6()); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-update", "dhcp6", + arguments)); + ElementPtr lease_as_json = leaseAsJson(createLease6()); + // The lease update must contain the "force-create" parameter indicating that + // the lease must be created if it doesn't exist. + lease_as_json->set("force-create", Element::create(true)); + lease_as_json->set("origin", Element::create("ha-partner")); + EXPECT_EQ(lease_as_json->str(), arguments->str()); +} + +// This test verifies that the command generated for the lease deletion +// is correct. +TEST(CommandCreatorTest, createLease6Delete) { + ConstElementPtr command = CommandCreator::createLease6Delete(*createLease6()); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-del", "dhcp6", + arguments)); + ElementPtr lease_as_json = leaseAsJson(createLease6()); + lease_as_json->set("origin", Element::create("ha-partner")); + EXPECT_EQ(lease_as_json->str(), arguments->str()); +} + +// This test verifies that the lease6-bulk-apply command is correct. +TEST(CommandCreatorTest, createLease6BulkApply) { + Lease6Ptr lease = createLease6(); + Lease6Ptr deleted_lease = createLease6(); + + Lease6CollectionPtr leases(new Lease6Collection()); + Lease6CollectionPtr deleted_leases(new Lease6Collection()); + + leases->push_back(lease); + deleted_leases->push_back(lease); + + ConstElementPtr command = CommandCreator::createLease6BulkApply(leases, deleted_leases); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-bulk-apply", + "dhcp6", arguments)); + + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); + + // Verify deleted-leases. + auto deleted_leases_json = arguments->get("deleted-leases"); + ASSERT_TRUE(deleted_leases_json); + ASSERT_EQ(Element::list, deleted_leases_json->getType()); + ASSERT_EQ(1, deleted_leases_json->size()); + auto lease_as_json = deleted_leases_json->get(0); + EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str()); + + // Verify leases. + auto leases_json = arguments->get("leases"); + ASSERT_TRUE(leases_json); + ASSERT_EQ(Element::list, leases_json->getType()); + ASSERT_EQ(1, leases_json->size()); + lease_as_json = leases_json->get(0); + EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str()); +} + +// This test verifies that the lease6-bulk-apply command can be created +// from DHCPv6 leases backlog. +TEST(CommandCreatorTest, createLease6BulkApplyFromBacklog) { + Lease6Ptr lease = createLease6(); + Lease6Ptr deleted_lease = createLease6(); + + LeaseUpdateBacklog backlog(100); + backlog.push(LeaseUpdateBacklog::ADD, lease); + backlog.push(LeaseUpdateBacklog::DELETE, deleted_lease); + + ConstElementPtr command = CommandCreator::createLease6BulkApply(backlog); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-bulk-apply", + "dhcp6", arguments)); + + ConstElementPtr origin = arguments->get("origin"); + ASSERT_TRUE(origin); + ASSERT_EQ("ha-partner", origin->stringValue()); + + // Verify deleted-leases. + auto deleted_leases_json = arguments->get("deleted-leases"); + ASSERT_TRUE(deleted_leases_json); + ASSERT_EQ(Element::list, deleted_leases_json->getType()); + ASSERT_EQ(1, deleted_leases_json->size()); + auto lease_as_json = deleted_leases_json->get(0); + EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str()); + + // Verify leases. + auto leases_json = arguments->get("leases"); + ASSERT_TRUE(leases_json); + ASSERT_EQ(Element::list, leases_json->getType()); + ASSERT_EQ(1, leases_json->size()); + lease_as_json = leases_json->get(0); + EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str()); + + // Make sure the backlog is now empty. + EXPECT_EQ(0, backlog.size()); +} + +// This test verifies that the lease6-get-all command is correct. +TEST(CommandCreatorTest, createLease6GetAll) { + ConstElementPtr command = CommandCreator::createLease6GetAll(); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-all", "dhcp6")); +} + + +// This test verifies that the lease6-get-page command is correct when +// first page is fetched. +TEST(CommandCreatorTest, createLease6GetPageStart) { + Lease6Ptr lease6; + ConstElementPtr command = CommandCreator::createLease6GetPage(lease6, 10); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-page", "dhcp6", + arguments)); + + ConstElementPtr from = arguments->get("from"); + ASSERT_TRUE(from); + EXPECT_EQ(Element::string, from->getType()); + EXPECT_EQ("start", from->stringValue()); + + ConstElementPtr limit = arguments->get("limit"); + ASSERT_TRUE(limit); + ASSERT_EQ(Element::integer, limit->getType()); + EXPECT_EQ(10, limit->intValue()); +} + +// This test verifies that the lease6-get-page command is correct when next +// page is fetched. +TEST(CommandCreatorTest, createLease6GetPageAddress) { + Lease6Ptr lease6(new Lease6()); + lease6->addr_ = IOAddress("2001:db8:1::1"); + + ConstElementPtr command = CommandCreator::createLease6GetPage(lease6, 15); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-page", "dhcp6", + arguments)); + + ConstElementPtr from = arguments->get("from"); + ASSERT_TRUE(from); + EXPECT_EQ(Element::string, from->getType()); + EXPECT_EQ("2001:db8:1::1", from->stringValue()); + + ConstElementPtr limit = arguments->get("limit"); + ASSERT_TRUE(limit); + ASSERT_EQ(Element::integer, limit->getType()); + EXPECT_EQ(15, limit->intValue()); +} + +// This test verifies that exception is thrown if limit is set to 0 while +// creating lease6-get-page command. +TEST(CommandCreatorTest, createLease6GetPageZeroLimit) { + Lease6Ptr lease6; + EXPECT_THROW(CommandCreator::createLease6GetPage(lease6, 0), BadValue); +} + +// This test verifies that the ha-maintenance-notify command is correct +// while being sent to the DHCPv4 server. +TEST(CommandCreatorTest, createMaintenanceNotify4) { + ConstElementPtr command = CommandCreator::createMaintenanceNotify(true, HAServerType::DHCPv4); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp4", + arguments)); + + auto cancel = arguments->get("cancel"); + ASSERT_TRUE(cancel); + ASSERT_EQ(Element::boolean, cancel->getType()); + EXPECT_TRUE(cancel->boolValue()); +} + +// This test verifies that the ha-maintenance-notify command is correct +// while being sent to the DHCPv6 server. +TEST(CommandCreatorTest, createMaintenanceNotify6) { + ConstElementPtr command = CommandCreator::createMaintenanceNotify(false, HAServerType::DHCPv6); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp6", + arguments)); + + auto cancel = arguments->get("cancel"); + ASSERT_TRUE(cancel); + ASSERT_EQ(Element::boolean, cancel->getType()); + EXPECT_FALSE(cancel->boolValue()); +} + +// This test verifies that the ha-sync-complete-notify command sent to a +// DHCPv4 server is correct. +TEST(CommandCreatorTest, createSyncCompleteNotify4) { + ConstElementPtr command = CommandCreator::createSyncCompleteNotify(HAServerType::DHCPv4); + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-sync-complete-notify", "dhcp4")); +} + +// This test verifies that the ha-sync-complete-notify command sent to a +// DHCPv4 server is correct. +TEST(CommandCreatorTest, createSyncCompleteNotify6) { + ConstElementPtr command = CommandCreator::createSyncCompleteNotify(HAServerType::DHCPv6); + ConstElementPtr arguments; + ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-sync-complete-notify", "dhcp6")); +} + +} diff --git a/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc new file mode 100644 index 0000000..8c33c09 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc @@ -0,0 +1,1201 @@ +// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace isc; +using namespace isc::asiolink; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::ha; +using namespace isc::ha::test; +using namespace isc::http; +using namespace isc::util; + +using namespace boost::posix_time; +using namespace boost::gregorian; + + +namespace { + + +/// @brief Test fixture class for @c CommunicationState class. +class CommunicationStateTest : public HATest { +public: + + /// @brief Constructor. + CommunicationStateTest() + : state_(io_service_, createValidConfiguration()), + state6_(io_service_, createValidConfiguration()) { + MultiThreadingMgr::instance().setMode(false); + } + + /// @brief Destructor. + ~CommunicationStateTest() { + MultiThreadingMgr::instance().setMode(false); + io_service_->poll(); + } + + /// @brief Verifies that the partner state is set and retrieved correctly. + void partnerStateTest(); + + /// @brief Verifies that the partner state is set to unavailable. + /// + /// Whether or not the function under test also resets the clock skew is + /// tested in a different test case. + void partnerStateUnavailableTest(); + + /// @brief Verifies that the partner's scopes are set and retrieved correctly. + void partnerScopesTest(); + + /// @brief Verifies that the object is poked right after construction. + void initialDurationTest(); + + /// @brief Verifies that poking the state updates the returned duration. + void pokeTest(); + + /// @brief Test that heartbeat function is triggered. + void heartbeatTest(); + + /// @brief Test that invalid values provided to startHeartbeat are rejected. + void startHeartbeatInvalidValuesTest(); + + /// @brief Test that failure detection works properly for DHCPv4 case. + void detectFailureV4Test(); + + /// @brief This test verifies that it is possible to disable analysis of the DHCPv4 + /// packets in which case the partner's failure is assumed when there is + /// no connection over the control channel. + void failureDetectionDisabled4Test(); + + /// @brief Test that failure detection works properly for DHCPv6 case. + void detectFailureV6Test(); + + /// @brief This test verifies that it is possible to disable analysis of the DHCPv6 + /// packets in which case the partner's failure is assumed when there is + /// no connection over the control channel. + void failureDetectionDisabled6Test(); + + /// @brief This test verifies that the clock skew is checked properly by the + /// clockSkewShouldWarn and clockSkewShouldTerminate functions. + void clockSkewTest(); + + /// @brief This test verifies that the clock skew calculations take into + /// account whether or not the partner is available. + void clockSkewPartnerUnavailableTest(); + + /// @brief This test verifies that the clock skew value is formatted correctly + /// for logging. + void logFormatClockSkewTest(); + + /// @brief This test verifies that too many rejected lease updates cause + /// the service termination. + void rejectedLeaseUpdatesTerminateTest(); + + /// @brief Tests that the communication state report is correct. + void getReportTest(); + + /// @brief Tests unusual values used to create the report. + void getReportDefaultValuesTest(); + + /// @brief Tests that unsent updates count can be incremented and fetched. + void getUnsentUpdateCountTest(); + + /// @brief Tests that unsent updates count from partner can be set and + /// a difference from previous value detected. + void hasPartnerNewUnsentUpdatesTest(); + + /// @brief Test that gathering rejected leases works fine in DHCPv4 case. + void reportRejectedLeasesV4Test(); + + /// @brief Test that rejected leases are cleared after reporting respective + /// successful leases. + void reportSuccessfulLeasesV4Test(); + + /// @brief Test that invalid values are not accepted when reporting + /// rejected leases. + void reportRejectedLeasesV4InvalidValuesTest(); + + /// @brief Test that gathering rejected leases works fine in the DHCPv6 case. + void reportRejectedLeasesV6Test(); + + /// @brief Test that rejected leases are cleared after reporting respective + /// successful leases. + void reportSuccessfulLeasesV6Test(); + + /// @brief Test that invalid values are not accepted when reporting + /// rejected leases. + void reportRejectedLeasesV6InvalidValuesTest(); + + /// @brief Test that old rejected lease updates are discarded while + /// getting the rejected lease updates count. + void getRejectedLeaseUpdatesCountFromContainerTest(); + + /// @brief Returns test heartbeat implementation. + /// + /// @return Pointer to heartbeat implementation function under test. + std::function getHeartbeatImpl() { + return (std::bind(&CommunicationStateTest::heartbeatImpl, this)); + } + + /// @brief Test heartbeat implementation. + /// + /// It simply pokes the communication state object. Note that the real + /// implementation would send an actual heartbeat command prior to + /// poking the state. + void heartbeatImpl() { + state_.poke(); + } + + /// @brief Communication state object used throughout the tests. + NakedCommunicationState4 state_; + + /// @brief Communication state for IPv6 used throughout the tests. + NakedCommunicationState6 state6_; +}; + +// Verifies that the partner state is set and retrieved correctly. +void +CommunicationStateTest::partnerStateTest() { + // Initially the state is unknown. + EXPECT_LT(state_.getPartnerState(), 0); + + state_.setPartnerState("hot-standby"); + EXPECT_EQ(HA_HOT_STANDBY_ST, state_.getPartnerState()); + + state_.setPartnerState("load-balancing"); + EXPECT_EQ(HA_LOAD_BALANCING_ST, state_.getPartnerState()); + + state_.setPartnerState("partner-down"); + EXPECT_EQ(HA_PARTNER_DOWN_ST, state_.getPartnerState()); + + state_.setPartnerState("ready"); + EXPECT_EQ(HA_READY_ST, state_.getPartnerState()); + + state_.setPartnerState("syncing"); + EXPECT_EQ(HA_SYNCING_ST, state_.getPartnerState()); + + state_.setPartnerState("terminated"); + EXPECT_EQ(HA_TERMINATED_ST, state_.getPartnerState()); + + state_.setPartnerState("waiting"); + EXPECT_EQ(HA_WAITING_ST, state_.getPartnerState()); + + state_.setPartnerState("unavailable"); + EXPECT_EQ(HA_UNAVAILABLE_ST, state_.getPartnerState()); + + // An attempt to set unsupported value should result in exception. + EXPECT_THROW(state_.setPartnerState("unsupported"), BadValue); +} + +// Verifies that the partner state is set to unavailable. +void +CommunicationStateTest::partnerStateUnavailableTest() { + // Initially the state is unknown. + EXPECT_LT(state_.getPartnerState(), 0); + + // Set a valid state initially. + state_.setPartnerState("hot-standby"); + EXPECT_EQ(HA_HOT_STANDBY_ST, state_.getPartnerState()); + + state_.setPartnerUnavailable(); + EXPECT_EQ(HA_UNAVAILABLE_ST, state_.getPartnerState()); +} + +// Verifies that the partner's scopes are set and retrieved correctly. +void +CommunicationStateTest::partnerScopesTest() { + // Initially, the scopes should be empty. + ASSERT_TRUE(state_.getPartnerScopes().empty()); + + // Set new partner scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ \"server1\", \"server2\" ]")) + ); + + // Get them back. + auto returned = state_.getPartnerScopes(); + EXPECT_EQ(2, returned.size()); + EXPECT_EQ(1, returned.count("server1")); + EXPECT_EQ(1, returned.count("server2")); + + // Override the scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ \"server1\" ]")) + ); + returned = state_.getPartnerScopes(); + EXPECT_EQ(1, returned.size()); + EXPECT_EQ(1, returned.count("server1")); + + // Clear the scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ ]")) + ); + returned = state_.getPartnerScopes(); + EXPECT_TRUE(returned.empty()); + + // An attempt to set invalid JSON should fail. + EXPECT_THROW(state_.setPartnerScopes(Element::fromJSON("{ \"not-a-list\": 1 }")), + BadValue); +} + +// Verifies that the object is poked right after construction. +void +CommunicationStateTest::initialDurationTest() { + EXPECT_TRUE(state_.isPoked()); +} + +// Verifies that poking the state updates the returned duration. +void +CommunicationStateTest::pokeTest() { + state_.modifyPokeTime(-30); + ASSERT_GE(state_.getDurationInMillisecs(), 30000); + ASSERT_TRUE(state_.isCommunicationInterrupted()); + ASSERT_NO_THROW(state_.poke()); + EXPECT_TRUE(state_.isPoked()); + EXPECT_FALSE(state_.isCommunicationInterrupted()); +} + +// Test that heartbeat function is triggered. +void +CommunicationStateTest::heartbeatTest() { + // Set poke time to the past and expect that the object is considered + // not poked. + state_.modifyPokeTime(-30); + EXPECT_FALSE(state_.isPoked()); + + // Run heartbeat every 1 second. + ASSERT_NO_THROW(state_.startHeartbeat(1, getHeartbeatImpl())); + runIOService(1200); + + // After > than 1 second the state should have been poked. + EXPECT_TRUE(state_.isPoked()); + + // Repeat the test. + state_.modifyPokeTime(-30); + EXPECT_FALSE(state_.isPoked()); + ASSERT_NO_THROW(state_.startHeartbeat(1, getHeartbeatImpl())); + runIOService(1200); + EXPECT_TRUE(state_.isPoked()); +} + +// Test that invalid values provided to startHeartbeat are rejected. +void +CommunicationStateTest::startHeartbeatInvalidValuesTest() { + EXPECT_THROW(state_.startHeartbeat(-1, getHeartbeatImpl()), BadValue); + EXPECT_THROW(state_.startHeartbeat(0, getHeartbeatImpl()), BadValue); + EXPECT_THROW(state_.startHeartbeat(1, 0), BadValue); +} + +// Test that failure detection works properly for DHCPv4 case. +void +CommunicationStateTest::detectFailureV4Test() { + // Initially, there should be no unacked clients recorded. + ASSERT_FALSE(state_.failureDetected()); + EXPECT_EQ(0, state_.getUnackedClientsCount()); + EXPECT_EQ(0, state_.getConnectingClientsCount()); + EXPECT_EQ(0, state_.getAnalyzedMessagesCount()); + + // The maximum number of unacked clients is 10. Let's provide 10 + // DHCPDISCOVER messages with the "secs" value of 15 which exceeds + // the threshold of 10. All these clients should be recorded as + // unacked. + for (uint8_t i = 0; i < 10; ++i) { + // Some of the requests have no client identifier to test that + // we don't fall over if the client identifier is null. + const uint8_t client_id_seed = (i < 5 ? i : 0); + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, + client_id_seed, + 15))); + // We don't exceed the maximum of number of unacked clients so the + // partner failure shouldn't be reported. + ASSERT_FALSE(state_.failureDetected()) + << "failure detected for the request number " + << static_cast(i); + } + EXPECT_EQ(10, state_.getUnackedClientsCount()); + EXPECT_EQ(10, state_.getConnectingClientsCount()); + EXPECT_EQ(10, state_.getAnalyzedMessagesCount()); + + // Let's provide similar set of requests but this time the "secs" field is + // below the threshold. They should not be counted as failures. Also, + // all of these requests have client identifier. + for (uint8_t i = 0; i < 10; ++i) { + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i, + 9))); + ASSERT_FALSE(state_.failureDetected()) + << "failure detected for the request number " + << static_cast(i); + } + EXPECT_EQ(10, state_.getUnackedClientsCount()); + EXPECT_EQ(15, state_.getConnectingClientsCount()); + EXPECT_EQ(20, state_.getAnalyzedMessagesCount()); + + // Let's create a message from a new (not recorded yet) client with the + // "secs" field value below the threshold. It should not be counted as failure. + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 10, 10, 6))); + + // Still no failure. + ASSERT_FALSE(state_.failureDetected()); + EXPECT_EQ(10, state_.getUnackedClientsCount()); + EXPECT_EQ(16, state_.getConnectingClientsCount()); + EXPECT_EQ(21, state_.getAnalyzedMessagesCount()); + + // Let's repeat one of the requests which already have been recorded as + // unacked but with a greater value of "secs" field. This should not + // be counted because only new clients count. + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 3, 3, 20))); + ASSERT_FALSE(state_.failureDetected()); + EXPECT_EQ(10, state_.getUnackedClientsCount()); + EXPECT_EQ(16, state_.getConnectingClientsCount()); + EXPECT_EQ(22, state_.getAnalyzedMessagesCount()); + + // This time let's simulate a client with a MAC address already recorded but + // with a client identifier. This should be counted as a new unacked request. + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 7, 7, 15))); + ASSERT_TRUE(state_.failureDetected()); + EXPECT_EQ(11, state_.getUnackedClientsCount()); + EXPECT_EQ(16, state_.getConnectingClientsCount()); + EXPECT_EQ(23, state_.getAnalyzedMessagesCount()); + + // Poking should cause all counters to reset as it is an indication that the + // control connection has been re-established. + ASSERT_NO_THROW(state_.poke()); + + // We're back to no failure state. + EXPECT_FALSE(state_.failureDetected()); + EXPECT_EQ(0, state_.getUnackedClientsCount()); + EXPECT_EQ(0, state_.getConnectingClientsCount()); + EXPECT_EQ(0, state_.getAnalyzedMessagesCount()); + + // Send 11 DHCPDISCOVER messages with the "secs" field bytes swapped. Swapping + // bytes was reported for some misbehaving Windows clients. The server should + // detect bytes swapping when second byte is 0 and the first byte is non-zero. + // However, the first byte is equal to 5 which is below our threshold so none + // of the requests below should count as unacked. + for (uint8_t i = 0; i < 11; ++i) { + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i, + 0x0500))); + ASSERT_FALSE(state_.failureDetected()) + << "failure detected for the request number " + << static_cast(i) + << " when testing swapped secs field bytes"; + } + EXPECT_EQ(0, state_.getUnackedClientsCount()); + EXPECT_EQ(11, state_.getConnectingClientsCount()); + EXPECT_EQ(11, state_.getAnalyzedMessagesCount()); + + // Repeat the same test, but this time either the first byte exceeds the + // secs threshold or the second byte is non-zero. All should be counted + // as unacked. + for (uint8_t i = 0; i < 10; ++i) { + uint16_t secs = (i % 2 == 0 ? 0x0F00 : 0x0501); + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i, + secs))); + ASSERT_FALSE(state_.failureDetected()) + << "failure detected for the request number " + << static_cast(i) + << " when testing swapped secs field bytes"; + } + + // This last message should cause the failure state. + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 11, 11, + 0x30))); + EXPECT_TRUE(state_.failureDetected()); + EXPECT_EQ(11, state_.getUnackedClientsCount()); + EXPECT_EQ(12, state_.getConnectingClientsCount()); + EXPECT_EQ(22, state_.getAnalyzedMessagesCount()); +} + +// This test verifies that it is possible to disable analysis of the DHCPv4 +// packets in which case the partner's failure is assumed when there is +// no connection over the control channel. +void +CommunicationStateTest::failureDetectionDisabled4Test() { + state_.config_->setMaxUnackedClients(0); + EXPECT_TRUE(state_.failureDetected()); +} + +// Test that failure detection works properly for DHCPv6 case. +void +CommunicationStateTest::detectFailureV6Test() { + // Initially, there should be no unacked clients recorded. + ASSERT_FALSE(state6_.failureDetected()); + EXPECT_EQ(0, state6_.getUnackedClientsCount()); + EXPECT_EQ(0, state6_.getConnectingClientsCount()); + EXPECT_EQ(0, state6_.getAnalyzedMessagesCount()); + + // The maximum number of unacked clients is 10. Let's provide 10 + // Solicit messages with the "elapsed time" value of 1500 which exceeds + // the threshold of 10000ms. Note that the elapsed time value is provided + // in 1/100s of 1 second. All these clients should be recorded as + // unacked. + for (uint8_t i = 0; i < 10; ++i) { + ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, i, + 1500))); + // We don't exceed the maximum number of unacked clients so the + // partner failure shouldn't be reported. + ASSERT_FALSE(state6_.failureDetected()) + << "failure detected for the request number " + << static_cast(i); + } + EXPECT_EQ(10, state6_.getUnackedClientsCount()); + EXPECT_EQ(10, state6_.getConnectingClientsCount()); + EXPECT_EQ(10, state6_.getAnalyzedMessagesCount()); + + // Let's provide similar set of requests but this time the "elapsed time" is + // below the threshold. This should not reduce the number of unacked or new + // clients. + for (uint8_t i = 0; i < 10; ++i) { + ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, i, + 900))); + ASSERT_FALSE(state6_.failureDetected()) + << "failure detected for the request number " + << static_cast(i); + } + EXPECT_EQ(10, state6_.getUnackedClientsCount()); + EXPECT_EQ(10, state6_.getConnectingClientsCount()); + EXPECT_EQ(20, state6_.getAnalyzedMessagesCount()); + + // Let's create a message from a new (not recorded yet) client with the + // "elapsed time" value below the threshold. It should not count as failure. + ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 10, 600))); + + // Still no failure. + ASSERT_FALSE(state6_.failureDetected()); + EXPECT_EQ(10, state6_.getUnackedClientsCount()); + EXPECT_EQ(11, state6_.getConnectingClientsCount()); + EXPECT_EQ(21, state6_.getAnalyzedMessagesCount()); + + // Let's repeat one of the requests which already have been recorded as + // unacked but with a greater value of "elapsed time". This should not + // be counted because only new clients count. + ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 3, 2000))); + ASSERT_FALSE(state6_.failureDetected()); + EXPECT_EQ(10, state6_.getUnackedClientsCount()); + EXPECT_EQ(11, state6_.getConnectingClientsCount()); + EXPECT_EQ(22, state6_.getAnalyzedMessagesCount()); + + // New unacked client should cause failure to be detected. + ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 11, 1500))); + ASSERT_TRUE(state6_.failureDetected()); + EXPECT_EQ(11, state6_.getUnackedClientsCount()); + EXPECT_EQ(12, state6_.getConnectingClientsCount()); + EXPECT_EQ(23, state6_.getAnalyzedMessagesCount()); + + // Poking should cause all counters to reset as it is an indication that the + // control connection has been re-established. + ASSERT_NO_THROW(state6_.poke()); + + // We're back to no failure state. + EXPECT_FALSE(state6_.failureDetected()); + EXPECT_EQ(0, state6_.getUnackedClientsCount()); + EXPECT_EQ(0, state6_.getConnectingClientsCount()); + EXPECT_EQ(0, state6_.getAnalyzedMessagesCount()); +} + +// This test verifies that it is possible to disable analysis of the DHCPv6 +// packets in which case the partner's failure is assumed when there is +// no connection over the control channel. +void +CommunicationStateTest::failureDetectionDisabled6Test() { + state6_.config_->setMaxUnackedClients(0); + EXPECT_TRUE(state6_.failureDetected()); +} + +// This test verifies that the clock skew is checked properly by the +// clockSkewShouldWarn and clockSkewShouldTerminate functions. +void +CommunicationStateTest::clockSkewTest() { + // Default clock skew is 0. + EXPECT_FALSE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + + // Partner time is ahead by 15s (no warning). + state_.clock_skew_ += boost::posix_time::time_duration(0, 0, 15); + EXPECT_FALSE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + + // Partner time is behind by 15s (no warning). + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ -= boost::posix_time::time_duration(0, 0, 15); + EXPECT_FALSE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + + // Partner time is ahead by 35s (warning). + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ += boost::posix_time::time_duration(0, 0, 35); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + + // Partner time is behind by 35s (warning). + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ -= boost::posix_time::time_duration(0, 0, 35); + state_.last_clock_skew_warn_ = boost::posix_time::ptime(); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + + // Due to the gating mechanism this should not return true the second + // time. + EXPECT_FALSE(state_.clockSkewShouldWarn()); + + // But should warn if the warning was issued more than 60 seconds ago. + state_.last_clock_skew_warn_ -= boost::posix_time::time_duration(0, 1, 30); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + + // Partner time is ahead by 65s (warning and terminate). + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ += boost::posix_time::time_duration(0, 1, 5); + state_.last_clock_skew_warn_ = boost::posix_time::ptime(); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + EXPECT_TRUE(state_.clockSkewShouldTerminate()); + + // Partner time is behind by 65s (warning and terminate). + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ -= boost::posix_time::time_duration(0, 1, 5); + state_.last_clock_skew_warn_ = boost::posix_time::ptime(); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + EXPECT_TRUE(state_.clockSkewShouldTerminate()); +} + +// This test verifies that the clock skew calculations take into +// account whether or not the partner is available. +void +CommunicationStateTest::clockSkewPartnerUnavailableTest() { + // Default clock skew is 0. + EXPECT_FALSE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); + + // Move the clock skew beyond the 60s limit. The alarms about the + // too high clock skew should be activated. + state_.setPartnerTime(HttpDateTime().rfc1123Format()); + state_.clock_skew_ += boost::posix_time::time_duration(0, 1, 5); + EXPECT_TRUE(state_.clockSkewShouldWarn()); + EXPECT_TRUE(state_.clockSkewShouldTerminate()); + + // Mark the partner as unavailable. It should reset the clock skew + // because we don't know the actual partner's time at the moment. + state_.setPartnerUnavailable(); + EXPECT_FALSE(state_.clockSkewShouldWarn()); + EXPECT_FALSE(state_.clockSkewShouldTerminate()); +} + +// This test verifies that the clock skew value is formatted correctly +// for logging. +void +CommunicationStateTest::logFormatClockSkewTest() { + // Make sure logFormatClockSkew() does not throw if called prior + // the first call to setPartnerTime(). + std::string log; + ASSERT_NO_THROW(log = state_.logFormatClockSkew()); + EXPECT_EQ(std::string("skew not initialized"), log); + + // Get current time. + boost::posix_time::ptime now = HttpDateTime().getPtime(); + + // Partner time is ahead by 15s. + boost::posix_time::time_duration offset(0,0,15); + state_.setPartnerTime(HttpDateTime(now + offset).rfc1123Format()); + ASSERT_NO_THROW(log = state_.logFormatClockSkew()); + + // The logFormatClockSkew uses the clock_skew_ value which is computed + // at the time when setPartnerTime() is called. Therefore, we can't + // just assume that it is 15s because it may be already slightly off. + // Let's compare the output with the actual clock_skew_ value remembered + // in the state_ instance. + ASSERT_FALSE(state_.clock_skew_.is_special()); + ASSERT_FALSE(state_.clock_skew_.is_negative()); + std::ostringstream s; + s << state_.clock_skew_.seconds() << "s ahead"; + EXPECT_TRUE(log.find(s.str()) != std::string::npos) << + " log content wrong: " << log; + + // Partner time is behind by 15s. + state_.setPartnerTime(HttpDateTime(now - offset).rfc1123Format()); + ASSERT_NO_THROW(log = state_.logFormatClockSkew()); + + // Again, extract the actual clock skew remembered in the state_ instance. + ASSERT_FALSE(state_.clock_skew_.is_special()); + auto skew = state_.clock_skew_; + + // It must be negative this time. + ASSERT_TRUE(skew.is_negative()); + // Convert it to positive value so we can use to to build the expected string. + skew = -skew; + std::ostringstream s2; + s2 << skew.seconds() << "s behind"; + EXPECT_TRUE(log.find(s2.str()) != std::string::npos) << + " log content wrong: " << log; + + offset = hours(18) + minutes(37) + seconds(15); + ptime mytime(date(2019, Jul, 23), offset); + + state_.my_time_at_skew_ = mytime; + state_.partner_time_at_skew_ = mytime + seconds(25); + state_.clock_skew_ = seconds(25); + ASSERT_NO_THROW(log = state_.logFormatClockSkew()); + std::string expected("my time: 2019-07-23 18:37:15, " + "partner's time: 2019-07-23 18:37:40, " + "partner's clock is 25s ahead"); + EXPECT_EQ(expected, log); +} + +void +CommunicationStateTest::rejectedLeaseUpdatesTerminateTest() { + EXPECT_FALSE(state_.rejectedLeaseUpdatesShouldTerminate()); + // Reject several lease updates but do not exceed the limit. + for (auto i = 0; i < 9; ++i) { + ASSERT_NO_THROW(state_.reportRejectedLeaseUpdate(createMessage4(DHCPDISCOVER, i, i, 0))); + } + EXPECT_FALSE(state_.rejectedLeaseUpdatesShouldTerminate()); + // Add one more. It should exceed the limit. + ASSERT_NO_THROW(state_.reportRejectedLeaseUpdate(createMessage4(DHCPDISCOVER, 9, 9, 0))); + EXPECT_TRUE(state_.rejectedLeaseUpdatesShouldTerminate()); +} + +// Tests that the communication state report is correct. +void +CommunicationStateTest::getReportTest() { + state_.setPartnerState("waiting"); + + auto scopes = Element::createList(); + scopes->add(Element::create("server1")); + state_.setPartnerScopes(scopes); + + state_.poke(); + + // Simulate the communications interrupted state. + state_.modifyPokeTime(-100); + + // Send two DHCP packets of which one has secs value beyond the threshold and + // the other one lower than the threshold. + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 0, 0, 5))); + ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 1, 0, 15))); + + // Get the report. + auto report = state_.getReport(); + ASSERT_TRUE(report); + + // Compare with the expected output. + std::string expected = "{" + " \"age\": 100," + " \"in-touch\": true," + " \"last-scopes\": [ \"server1\" ]," + " \"last-state\": \"waiting\"," + " \"communication-interrupted\": true," + " \"connecting-clients\": 2," + " \"unacked-clients\": 1," + " \"unacked-clients-left\": 10," + " \"analyzed-packets\": 2" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), report)); +} + +// Tests unusual values used to create the report. +void +CommunicationStateTest::getReportDefaultValuesTest() { + auto report = state_.getReport(); + ASSERT_TRUE(report); + + // Compare with the expected output. + std::string expected = "{" + " \"age\": 0," + " \"in-touch\": false," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"," + " \"communication-interrupted\": false," + " \"connecting-clients\": 0," + " \"unacked-clients\": 0," + " \"unacked-clients-left\": 0," + " \"analyzed-packets\": 0" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), report)); +} + +void +CommunicationStateTest::getUnsentUpdateCountTest() { + // Initially the count should be 0. + EXPECT_EQ(0, state_.getUnsentUpdateCount()); + + // Increasing the value by 1 several times. + EXPECT_NO_THROW(state_.increaseUnsentUpdateCount()); + EXPECT_EQ(1, state_.getUnsentUpdateCount()); + EXPECT_NO_THROW(state_.increaseUnsentUpdateCount()); + EXPECT_EQ(2, state_.getUnsentUpdateCount()); + EXPECT_NO_THROW(state_.increaseUnsentUpdateCount()); + EXPECT_EQ(3, state_.getUnsentUpdateCount()); + + // Test that the method under test protects against an overflow + // resetting the value to 0. + state_.unsent_update_count_ = std::numeric_limits::max(); + EXPECT_NO_THROW(state_.increaseUnsentUpdateCount()); + EXPECT_EQ(1, state_.getUnsentUpdateCount()); +} + +void +CommunicationStateTest::hasPartnerNewUnsentUpdatesTest() { + // Initially the counts should be 0. + EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates()); + + // Set a positive value. It should be noticed. + EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(5)); + EXPECT_TRUE(state_.hasPartnerNewUnsentUpdates()); + + // No change, no new unsent updates. + EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(5)); + EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates()); + + // Change it again. New updates. + EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(10)); + EXPECT_TRUE(state_.hasPartnerNewUnsentUpdates()); + + // Set it to 0 to simulate restart. No updates. + EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(0)); + EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates()); +} + +void +CommunicationStateTest::reportRejectedLeasesV4Test() { + // Initially, there should be no rejected leases. + EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount()); + + // Reject lease update. + auto msg = createMessage4(DHCPREQUEST, 1, 0, 0); + state_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount()); + + // Reject another lease update. + msg = createMessage4(DHCPREQUEST, 2, 0, 0); + state_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount()); + + // Reject a lease with a short (zero) lease lifetime. + // This lease should be discarded when we call the + // getRejectedLeaseUpdatesCount(). + msg = createMessage4(DHCPREQUEST, 3, 0, 0); + state_.reportRejectedLeaseUpdate(msg, 0); + EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount()); + + // Reject lease update for a client using the same MAC + // address but different client identifier. It should + // be treated as a different lease. + msg = createMessage4(DHCPREQUEST, 2, 1, 0); + state_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(3, state_.getRejectedLeaseUpdatesCount()); + + // Clear rejected leases and make sure the counter + // is now 0. + state_.clearRejectedLeaseUpdates(); + EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount()); +} + +void +CommunicationStateTest::reportSuccessfulLeasesV4Test() { + // Initially, there should be no rejected leases. + EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount()); + auto msg0 = createMessage4(DHCPREQUEST, 1, 0, 0); + // Reject lease update. + state_.reportRejectedLeaseUpdate(msg0); + EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount()); + // Reject another lease update. + auto msg1 = createMessage4(DHCPREQUEST, 2, 0, 0); + state_.reportRejectedLeaseUpdate(msg1); + EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount()); + // Report successful lease for the first message. + // It should reduce the number of rejected lease + // updates. + EXPECT_TRUE(state_.reportSuccessfulLeaseUpdate(msg0)); + EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount()); + // Report successful lease update for another message. + auto msg2 = createMessage4(DHCPREQUEST, 1, 1, 0); + EXPECT_FALSE(state_.reportSuccessfulLeaseUpdate(msg2)); + EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount()); + // There should be no rejected lease updates. + EXPECT_TRUE(state_.reportSuccessfulLeaseUpdate(msg1)); + EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount()); +} + +void +CommunicationStateTest::reportRejectedLeasesV4InvalidValuesTest() { + // Populate one valid update. Without it our functions under test + // would return early. + state_.reportRejectedLeaseUpdate(createMessage4(DHCPREQUEST, 1, 0, 0)); + // Using DHCPv6 message in the DHCPv4 context is a programming + // error and deserves an exception. + auto msg = createMessage6(DHCPV6_REQUEST, 1, 0); + EXPECT_THROW(state_.reportRejectedLeaseUpdate(msg), BadValue); + EXPECT_THROW(state_.reportSuccessfulLeaseUpdate(msg), BadValue); +} + +void +CommunicationStateTest::reportRejectedLeasesV6Test() { + // Initially, there should be no rejected leases. + EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount()); + + // Reject lease update. + auto msg = createMessage6(DHCPV6_REQUEST, 1, 0); + state6_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount()); + + // Reject another lease update. + msg = createMessage6(DHCPV6_REQUEST, 2, 0); + state6_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount()); + + // Reject a lease with a short (zero) lease lifetime. + // This lease should be discarded when we call the + // getRejectedLeaseUpdatesCount(). + msg = createMessage6(DHCPV6_REQUEST, 3, 0); + state6_.reportRejectedLeaseUpdate(msg, 0); + EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount()); + + // Reject it again. It should not affect the counter. + msg = createMessage6(DHCPV6_REQUEST, 2, 0); + state6_.reportRejectedLeaseUpdate(msg); + EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount()); + + // Clear rejected lease updates and make sure they + // are now 0. + state6_.clearRejectedLeaseUpdates(); + EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount()); +} + +void +CommunicationStateTest::reportSuccessfulLeasesV6Test() { + // Initially, there should be no rejected leases. + EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount()); + // Reject lease update. + auto msg0 = createMessage6(DHCPV6_SOLICIT, 1, 0); + EXPECT_TRUE(state6_.reportRejectedLeaseUpdate(msg0)); + EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount()); + // Reject another lease update. + auto msg1 = createMessage6(DHCPV6_SOLICIT, 2, 0); + EXPECT_TRUE(state6_.reportRejectedLeaseUpdate(msg1)); + EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount()); + // Report successful lease for the first message. + // It should reduce the number of rejected lease + // updates. + EXPECT_TRUE(state6_.reportSuccessfulLeaseUpdate(msg0)); + EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount()); + // Report successful lease update for a lease that wasn't + // rejected. It should not affect the counter. + auto msg2 = createMessage6(DHCPV6_SOLICIT, 3, 0); + EXPECT_FALSE(state6_.reportSuccessfulLeaseUpdate(msg2)); + EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount()); + // Report successful lease update for the last lease. + // The counter should now be 0. + EXPECT_TRUE(state6_.reportSuccessfulLeaseUpdate(msg1)); + EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount()); +} + +void +CommunicationStateTest::reportRejectedLeasesV6InvalidValuesTest() { + // Populate one valid update. Without it our functions under test + // would return early. + state6_.reportRejectedLeaseUpdate(createMessage6(DHCPV6_REQUEST, 1, 0)); + // Using DHCPv4 message in the DHCPv6 context is a programming + // error and deserves an exception. + auto msg0 = createMessage4(DHCPREQUEST, 1, 1, 0); + EXPECT_THROW(state6_.reportRejectedLeaseUpdate(msg0), BadValue); + EXPECT_THROW(state6_.reportSuccessfulLeaseUpdate(msg0), BadValue); + + auto msg1 = createMessage6(DHCPV6_SOLICIT, 1, 0); + msg1->delOption(D6O_CLIENTID); + EXPECT_FALSE(state6_.reportRejectedLeaseUpdate(msg1)); + EXPECT_FALSE(state6_.reportSuccessfulLeaseUpdate(msg1)); +} + +void +CommunicationStateTest::getRejectedLeaseUpdatesCountFromContainerTest() { + // Create a simple multi index container with two indexes. The + // first index is on the ordinal number to distinguish between + // different entries. The second index is on the expire_ field + // that is identical to the expire_ field in the RejectedClients4 + // and RejectedClients6 containers. + struct Entry { + int64_t ordinal_; + int64_t expire_; + }; + typedef boost::multi_index_container< + Entry, + boost::multi_index::indexed_by< + boost::multi_index::ordered_unique< + boost::multi_index::member + >, + boost::multi_index::ordered_non_unique< + boost::multi_index::member + > + > + > Entries; + // Add many entries to the container. Odd entries have lifetime + // expiring in the future. Even entries have lifetimes expiring in + // the past. + Entries entries; + for (auto i = 0; i < 1000; i++) { + entries.insert({i, time(NULL) + (i % 2 ? 100 + i : -1 - i)}); + } + // Get the count of valid entries. It should remove the expiring + // entries. + auto valid_entries_count = state_.getRejectedLeaseUpdatesCountFromContainer(entries); + EXPECT_EQ(500, valid_entries_count); + EXPECT_EQ(500, entries.size()); + + // Validate that we removed expired entries, not the valid ones. + for (auto entry : entries) { + EXPECT_EQ(1, entry.ordinal_ % 2); + } +} + +TEST_F(CommunicationStateTest, partnerStateTest) { + partnerStateTest(); +} + +TEST_F(CommunicationStateTest, partnerStateTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + partnerStateTest(); +} + +TEST_F(CommunicationStateTest, partnerStateUnavailableTest) { + partnerStateUnavailableTest(); +} + +TEST_F(CommunicationStateTest, partnerStateUnavailableTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + partnerStateUnavailableTest(); +} + +TEST_F(CommunicationStateTest, partnerScopesTest) { + partnerScopesTest(); +} + +TEST_F(CommunicationStateTest, partnerScopesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + partnerScopesTest(); +} + +TEST_F(CommunicationStateTest, initialDurationTest) { + initialDurationTest(); +} + +TEST_F(CommunicationStateTest, initialDurationTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + initialDurationTest(); +} + +TEST_F(CommunicationStateTest, pokeTest) { + pokeTest(); +} + +TEST_F(CommunicationStateTest, pokeTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + pokeTest(); +} + +TEST_F(CommunicationStateTest, heartbeatTest) { + heartbeatTest(); +} + +TEST_F(CommunicationStateTest, heartbeatTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + heartbeatTest(); +} + +TEST_F(CommunicationStateTest, startHeartbeatInvalidValuesTest) { + startHeartbeatInvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, startHeartbeatInvalidValuesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + startHeartbeatInvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, detectFailureV4Test) { + detectFailureV4Test(); +} + +TEST_F(CommunicationStateTest, detectFailureV4TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + detectFailureV4Test(); +} + +TEST_F(CommunicationStateTest, failureDetectionDisabled4Test) { + failureDetectionDisabled4Test(); +} + +TEST_F(CommunicationStateTest, failureDetectionDisabled4TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + failureDetectionDisabled4Test(); +} + +TEST_F(CommunicationStateTest, detectFailureV6Test) { + detectFailureV6Test(); +} + +TEST_F(CommunicationStateTest, detectFailureV6TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + detectFailureV6Test(); +} + +TEST_F(CommunicationStateTest, failureDetectionDisabled6Test) { + failureDetectionDisabled6Test(); +} + +TEST_F(CommunicationStateTest, failureDetectionDisabled6TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + failureDetectionDisabled6Test(); +} + +TEST_F(CommunicationStateTest, clockSkewTest) { + clockSkewTest(); +} + +TEST_F(CommunicationStateTest, clockSkewTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + clockSkewTest(); +} + +TEST_F(CommunicationStateTest, clockSkewPartnerUnavailableTest) { + clockSkewPartnerUnavailableTest(); +} + +TEST_F(CommunicationStateTest, clockSkewPartnerUnavailableTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + clockSkewPartnerUnavailableTest(); +} + +TEST_F(CommunicationStateTest, rejectedLeaseUpdatesTerminateTest) { + rejectedLeaseUpdatesTerminateTest(); +} + +TEST_F(CommunicationStateTest, rejectedLeaseUpdatesTerminateTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + rejectedLeaseUpdatesTerminateTest(); +} + +TEST_F(CommunicationStateTest, logFormatClockSkewTest) { + logFormatClockSkewTest(); +} + +TEST_F(CommunicationStateTest, logFormatClockSkewTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + logFormatClockSkewTest(); +} + +TEST_F(CommunicationStateTest, getReportTest) { + getReportTest(); +} + +TEST_F(CommunicationStateTest, getReportTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + getReportTest(); +} + +TEST_F(CommunicationStateTest, getReportDefaultValuesTest) { + getReportDefaultValuesTest(); +} + +TEST_F(CommunicationStateTest, getReportDefaultValuesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + getReportDefaultValuesTest(); +} + +TEST_F(CommunicationStateTest, getUnsentUpdateCountTest) { + getUnsentUpdateCountTest(); +} + +TEST_F(CommunicationStateTest, getUnsentUpdateCountTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + getUnsentUpdateCountTest(); +} + +TEST_F(CommunicationStateTest, hasPartnerNewUnsentUpdatesTest) { + hasPartnerNewUnsentUpdatesTest(); +} + +TEST_F(CommunicationStateTest, hasPartnerNewUnsentUpdatesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hasPartnerNewUnsentUpdatesTest(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV4Test) { + reportRejectedLeasesV4Test(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV4TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportRejectedLeasesV4Test(); +} + +TEST_F(CommunicationStateTest, reportSuccessfulLeasesV4Test) { + reportSuccessfulLeasesV4Test(); +} + +TEST_F(CommunicationStateTest, reportSuccessfulLeasesV4TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportSuccessfulLeasesV4Test(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV4InvalidValuesTest) { + reportRejectedLeasesV4InvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV4InvalidValuesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportRejectedLeasesV4InvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV6Test) { + reportRejectedLeasesV6Test(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV6TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportRejectedLeasesV6Test(); +} + +TEST_F(CommunicationStateTest, reportSuccessfulLeasesV6Test) { + reportSuccessfulLeasesV6Test(); +} + +TEST_F(CommunicationStateTest, reportSuccessfulLeasesV6TestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportSuccessfulLeasesV6Test(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV6InvalidValuesTest) { + reportRejectedLeasesV6InvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, reportRejectedLeasesV6InvalidValuesTestMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + reportRejectedLeasesV6InvalidValuesTest(); +} + +TEST_F(CommunicationStateTest, getRejectedLeaseUpdatesCountFromContainerTest) { + getRejectedLeaseUpdatesCountFromContainerTest(); +} + +} diff --git a/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc new file mode 100644 index 0000000..f8e6940 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc @@ -0,0 +1,1879 @@ +// Copyright (C) 2017-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isc; +using namespace isc::config; +using namespace isc::data; +using namespace isc::ha; +using namespace isc::hooks; +using namespace isc::ha::test; +using namespace isc::util; + +namespace { + +/// @brief Test fixture class for testing HA hooks library +/// configuration. +class HAConfigTest : public HATest { +public: + /// @brief Constructor. + HAConfigTest() : HATest(), hardware_threads_(MultiThreadingMgr::detectThreadCount()) { + } + + /// @brief Verifies if an exception is thrown if provided HA + /// configuration is invalid. + /// + /// @param invalid_config Configuration to be tested. + /// @param expected_error Expected error message. + void testInvalidConfig(const std::string& invalid_config, + const std::string& expected_error) { + HAImplPtr impl(new HAImpl()); + try { + impl->configure(Element::fromJSON(invalid_config)); + ADD_FAILURE() << "expected ConfigError exception, thrown no exception"; + + } catch (const ConfigError& ex) { + // Expect the error to be contained in the exception message. + std::string const exception(ex.what()); + EXPECT_NE(exception.find(expected_error), std::string::npos); + } catch (...) { + ADD_FAILURE() << "expected ConfigError exception, thrown different" + " exception type"; + } + } + + /// @brief number of threads the system reports as supported + uint32_t hardware_threads_; +}; + +// Verifies that load balancing configuration is parsed correctly. +TEST_F(HAConfigTest, configureLoadBalancing) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"send-lease-updates\": false," + " \"sync-leases\": false," + " \"sync-timeout\": 20000," + " \"sync-page-limit\": 3," + " \"delayed-updates-limit\": 111," + " \"heartbeat-delay\": 8," + " \"max-response-delay\": 11," + " \"max-ack-delay\": 5," + " \"max-unacked-clients\": 20," + " \"max-rejected-lease-updates\": 9," + " \"wait-backup-ack\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"basic-auth-password\": \"1234\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"basic-auth-user\": \"\"," + " \"role\": \"secondary\"" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:8082/\"," + " \"basic-auth-user\": \"foo\"," + " \"basic-auth-password\": \"bar\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }" + " ]," + " \"state-machine\": {" + " \"states\": [" + " {" + " \"state\": \"waiting\"," + " \"pause\": \"once\"" + " }," + " {" + " \"state\": \"ready\"," + " \"pause\": \"always\"" + " }," + " {" + " \"state\": \"partner-down\"," + " \"pause\": \"never\"" + " }" + " ]" + " }" + " }" + "]"; + + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config))); + EXPECT_EQ("server1", impl->getConfig()->getThisServerName()); + EXPECT_EQ(HAConfig::LOAD_BALANCING, impl->getConfig()->getHAMode()); + EXPECT_FALSE(impl->getConfig()->amSendingLeaseUpdates()); + EXPECT_FALSE(impl->getConfig()->amSyncingLeases()); + EXPECT_EQ(20000, impl->getConfig()->getSyncTimeout()); + EXPECT_EQ(3, impl->getConfig()->getSyncPageLimit()); + EXPECT_EQ(111, impl->getConfig()->getDelayedUpdatesLimit()); + EXPECT_TRUE(impl->getConfig()->amAllowingCommRecovery()); + EXPECT_EQ(8, impl->getConfig()->getHeartbeatDelay()); + EXPECT_EQ(11, impl->getConfig()->getMaxResponseDelay()); + EXPECT_EQ(5, impl->getConfig()->getMaxAckDelay()); + EXPECT_EQ(20, impl->getConfig()->getMaxUnackedClients()); + EXPECT_EQ(9, impl->getConfig()->getMaxRejectedLeaseUpdates()); + EXPECT_FALSE(impl->getConfig()->amWaitingBackupAck()); + + HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig(); + ASSERT_TRUE(cfg); + EXPECT_EQ("server1", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText()); + EXPECT_EQ(cfg->getLogLabel(), "server1 (http://127.0.0.1:8080/)"); + EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole()); + EXPECT_FALSE(cfg->isAutoFailover()); + EXPECT_FALSE(cfg->getBasicAuth()); + + cfg = impl->getConfig()->getPeerConfig("server2"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server2", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText()); + EXPECT_EQ(cfg->getLogLabel(), "server2 (http://127.0.0.1:8081/)"); + EXPECT_EQ(HAConfig::PeerConfig::SECONDARY, cfg->getRole()); + EXPECT_TRUE(cfg->isAutoFailover()); + EXPECT_FALSE(cfg->getBasicAuth()); + + cfg = impl->getConfig()->getPeerConfig("server3"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server3", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText()); + EXPECT_EQ(cfg->getLogLabel(), "server3 (http://127.0.0.1:8082/)"); + EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole()); + EXPECT_FALSE(cfg->isAutoFailover()); + ASSERT_TRUE(cfg->getBasicAuth()); + EXPECT_EQ("foo:bar", cfg->getBasicAuth()->getSecret()); + + // Verify that per-state configuration is correct. + + HAConfig::StateConfigPtr state_cfg; + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_BACKUP_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_COMMUNICATION_RECOVERY_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_LOAD_BALANCING_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_PARTNER_DOWN_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_READY_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_ALWAYS, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_SYNCING_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_TERMINATED_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_WAITING_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_ONCE, state_cfg->getPausing()); + + // Verify multi-threading default values. Default is 0 for the listener and client threads, but + // after MT is applied, HAImpl resolves them to the auto-detected values. + EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading()); + EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads()); +} + +// Verifies that hot standby configuration is parsed correctly. +TEST_F(HAConfigTest, configureHotStandby) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"basic-auth-user\": \"admin\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:8082/\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }" + " ]" + " }" + "]"; + + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config))); + EXPECT_EQ("server1", impl->getConfig()->getThisServerName()); + EXPECT_EQ(HAConfig::HOT_STANDBY, impl->getConfig()->getHAMode()); + EXPECT_TRUE(impl->getConfig()->amSendingLeaseUpdates()); + EXPECT_TRUE(impl->getConfig()->amSyncingLeases()); + EXPECT_EQ(60000, impl->getConfig()->getSyncTimeout()); + EXPECT_EQ(10000, impl->getConfig()->getSyncPageLimit()); + EXPECT_EQ(0, impl->getConfig()->getDelayedUpdatesLimit()); + EXPECT_FALSE(impl->getConfig()->amAllowingCommRecovery()); + EXPECT_EQ(10000, impl->getConfig()->getHeartbeatDelay()); + EXPECT_EQ(10000, impl->getConfig()->getMaxAckDelay()); + EXPECT_EQ(10, impl->getConfig()->getMaxUnackedClients()); + EXPECT_EQ(10, impl->getConfig()->getMaxRejectedLeaseUpdates()); + EXPECT_FALSE(impl->getConfig()->amWaitingBackupAck()); + + HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig(); + ASSERT_TRUE(cfg); + EXPECT_EQ("server1", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole()); + EXPECT_FALSE(cfg->isAutoFailover()); + ASSERT_TRUE(cfg->getBasicAuth()); + EXPECT_EQ("admin:", cfg->getBasicAuth()->getSecret()); + + cfg = impl->getConfig()->getPeerConfig("server2"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server2", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::STANDBY, cfg->getRole()); + EXPECT_TRUE(cfg->isAutoFailover()); + EXPECT_FALSE(cfg->getBasicAuth()); + + cfg = impl->getConfig()->getPeerConfig("server3"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server3", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole()); + EXPECT_FALSE(cfg->isAutoFailover()); + EXPECT_FALSE(cfg->getBasicAuth()); + + HAConfig::StateConfigPtr state_cfg; + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_BACKUP_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_HOT_STANDBY_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_PARTNER_DOWN_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_READY_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_SYNCING_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_TERMINATED_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()-> + getStateConfig(HA_WAITING_ST)); + ASSERT_TRUE(state_cfg); + EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing()); + + // Verify multi-threading default values. Default is 0 for the listener and client threads, but + // after MT is applied, HAImpl resolves them to the auto-detected values. + EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading()); + EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads()); +} + +// Verifies that passive-backup configuration is parsed correctly. +TEST_F(HAConfigTest, configurePassiveBackup) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"wait-backup-ack\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"backup\"" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:8082/\"," + " \"basic-auth-user\": \"keatest\"," + " \"basic-auth-password-file\": \"" + TEST_HTTP_DIR "/hiddenp\"," + " \"role\": \"backup\"" + " }" + " ]" + " }" + "]"; + + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config))); + EXPECT_EQ("server1", impl->getConfig()->getThisServerName()); + EXPECT_EQ(HAConfig::PASSIVE_BACKUP, impl->getConfig()->getHAMode()); + EXPECT_TRUE(impl->getConfig()->amSendingLeaseUpdates()); + EXPECT_TRUE(impl->getConfig()->amWaitingBackupAck()); + + HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig(); + ASSERT_TRUE(cfg); + EXPECT_EQ("server1", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole()); + EXPECT_FALSE(cfg->getBasicAuth()); + + cfg = impl->getConfig()->getPeerConfig("server2"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server2", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole()); + EXPECT_FALSE(cfg->getBasicAuth()); + + cfg = impl->getConfig()->getPeerConfig("server3"); + ASSERT_TRUE(cfg); + EXPECT_EQ("server3", cfg->getName()); + EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText()); + EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole()); + ASSERT_TRUE(cfg->getBasicAuth()); + EXPECT_EQ("a2VhdGVzdDpLZWFUZXN0", cfg->getBasicAuth()->getCredential()); + + // Verify multi-threading default values. Default is 0 for the listener and client threads, but + // after MT is applied, HAImpl resolves them to the auto-detected values. + EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading()); + EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads()); + EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads()); +} + +// This server name must not be empty. +TEST_F(HAConfigTest, emptyServerName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'this-server-name' value must not be empty"); +} + +// There must be a configuration provided for this server. +TEST_F(HAConfigTest, nonMatchingServerName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"foo\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "no peer configuration specified for the 'foo'"); +} + +// Error should be returned when mode is invalid. +TEST_F(HAConfigTest, unsupportedMode) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"unsupported-mode\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "unsupported value 'unsupported-mode' for mode parameter"); +} + +// Error should be returned when heartbeat-delay is negative. +TEST_F(HAConfigTest, negativeHeartbeatDelay) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"heartbeat-delay\": -1," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'heartbeat-delay' must not be negative"); +} + +// Error should be returned when heartbeat-delay is too large. +TEST_F(HAConfigTest, largeHeartbeatDelay) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"heartbeat-delay\": 65536," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'heartbeat-delay' must not be greater than 65535"); +} + +// There must be at least two servers provided. +TEST_F(HAConfigTest, singlePeer) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }" + " ]" + " }" + "]", + "secondary server required in the load balancing configuration"); +} + +// Server name must not be empty. +TEST_F(HAConfigTest, emptyPeerName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "peer name must not be empty"); +} + +// Can't have two servers with the same name. +TEST_F(HAConfigTest, duplicatePeerName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "peer with name 'server1' already specified"); +} + +// URL must be valid. +TEST_F(HAConfigTest, invalidURL) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "invalid URL: url ://127.0.0.1:8080/ lacks http or" + " https scheme for server server2"); +} + +// URL hostname must be an address. +TEST_F(HAConfigTest, badURLName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://localhost:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "bad url 'http://localhost:8080/': " + "Failed to convert string to address 'localhost': " + "Invalid argument for server server2"); +} + +// URL HTTPS scheme is not supported. +TEST_F(HAConfigTest, badURLHttps) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"https://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "bad url 'https://127.0.0.1:8080/': " + "https scheme is not supported for server server2 " + "where TLS is disabled"); +} + +// Only certain roles are allowed. +TEST_F(HAConfigTest, unsupportedRole) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"unsupported\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "unsupported value 'unsupported' for role parameter"); +} + +// There must be exactly one primary server. +TEST_F(HAConfigTest, twoPrimary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "multiple primary servers specified"); +} + +// There must be exactly one secondary server. +TEST_F(HAConfigTest, twoSecondary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "multiple secondary servers specified"); +} + +// Only one standby server is allowed. +TEST_F(HAConfigTest, twoStandby) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"standby\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "multiple standby servers specified"); +} + +// Primary server is required for load balancing. +TEST_F(HAConfigTest, loadBalancingNoPrimary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "primary server required in the load balancing configuration"); +} + +// Secondary server is required for load balancing. +TEST_F(HAConfigTest, loadBalancingNoSecondary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "secondary server required in the load balancing configuration"); +} + +// Primary server is required for hot standby mode. +TEST_F(HAConfigTest, hotStandbyNoPrimary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "primary server required in the hot standby configuration"); +} + +// Standby server is required for hot standby mode. +TEST_F(HAConfigTest, hotStandbyNoStandby) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "standby server required in the hot standby configuration"); +} + +// Standby server must not be specified in the load balancing mode. +TEST_F(HAConfigTest, loadBalancingStandby) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "standby servers not allowed in the load balancing configuration"); +} + +// Secondary server must not be specified in the hot standby mode. +TEST_F(HAConfigTest, hotStandbySecondary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "secondary servers not allowed in the hot standby configuration"); +} + +// state-machine parameter must be a map. +TEST_F(HAConfigTest, invalidStateMachine) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]," + " \"state-machine\": [" + " {" + " \"state\": \"foo\"," + " \"pause\": \"always\"" + " }" + " ]" + " }" + "]", + "'state-machine' parameter must be a map"); +} + +// states within state-machine must be a list. +TEST_F(HAConfigTest, invalidStatesList) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]," + " \"state-machine\": {" + " \"states\": {" + " }" + " }" + " }" + "]", + "'states' parameter must be a list"); +} + +// State name must be recognized. +TEST_F(HAConfigTest, invalidStateName) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]," + " \"state-machine\": {" + " \"states\": [" + " {" + " \"state\": \"foo\"," + " \"pause\": \"always\"" + " }" + " ]" + " }" + " }" + "]", + "unknown state foo"); +} + +// Pause value must be recognized. +TEST_F(HAConfigTest, invalidPauseValue) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]," + " \"state-machine\": {" + " \"states\": [" + " {" + " \"state\": \"waiting\"," + " \"pause\": \"foo\"" + " }" + " ]" + " }" + " }" + "]", + "unsupported value foo of 'pause' parameter"); +} + +// Must not specify configuration for the same state twice. +TEST_F(HAConfigTest, duplicatedStates) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]," + " \"state-machine\": {" + " \"states\": [" + " {" + " \"state\": \"waiting\"," + " \"pause\": \"always\"" + " }," + " {" + " \"state\": \"ready\"," + " \"pause\": \"always\"" + " }," + " {" + " \"state\": \"waiting\"," + " \"pause\": \"always\"" + " }" + " ]" + " }" + " }" + "]", + "duplicated configuration for the 'waiting' state"); +} + +// Test that wait-backup-ack must not be enabled in the load-balancing +// configuration. +TEST_F(HAConfigTest, waitBackupAckLoadBalancing) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"wait-backup-ack\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'wait-backup-ack' must be set to false in the load balancing configuration"); +} + +// Test that wait-backup-ack must not be enabled in the hot-standby +// configuration. +TEST_F(HAConfigTest, waitBackupAckHotStandby) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"wait-backup-ack\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'wait-backup-ack' must be set to false in the hot standby configuration"); +} + +// Test that secondary server is not allowed in the passive-backup mode. +TEST_F(HAConfigTest, passiveBackupSecondaryServer) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"secondary\"" + " }" + " ]" + " }" + "]", + "secondary servers not allowed in the passive backup configuration"); +} + +// Test that standby server is not allowed in the passive-backup mode. +TEST_F(HAConfigTest, passiveBackupStandbyServer) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"standby\"" + " }" + " ]" + " }" + "]", + "standby servers not allowed in the passive backup configuration"); +} + +// Test that primary server is required in the passive-backup mode. +TEST_F(HAConfigTest, passiveBackupNoPrimary) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"backup\"" + " }" + " ]" + " }" + "]", + "primary server required in the passive backup configuration"); +} + +// Test that empty name id is forbidden for basic HTTP authentication. +TEST_F(HAConfigTest, invalidUser) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"basic-auth-user\": \"foo:bar\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "user 'foo:bar' must not contain a ':' in peer 'server2'"); +} + +// Test that setting delayed-updates-limit is not allowed in hot-standby mode. +TEST_F(HAConfigTest, hotStandbyDelayedUpdatesLimit) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"hot-standby\"," + " \"delayed-updates-limit\": 1," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"standby\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]", + "'delayed-updates-limit' must be set to 0 in the hot standby configuration"); +} + +// Test that setting delayed-updates-limit is not allowed in passive-backup mode. +TEST_F(HAConfigTest, passiveBackupDelayedUpdatesLimit) { + testInvalidConfig( + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"delayed-updates-limit\": 1," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"" + " }" + " ]" + " }" + "]", + "'delayed-updates-limit' must be set to 0 in the passive backup configuration"); +} + +#if (defined(WITH_OPENSSL) || defined(WITH_BOTAN_BOOST)) +/// Test that TLS parameters are correctly inherited. +TEST_F(HAConfigTest, tlsParameterInheritance) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"my-server\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"restrict-commands\": true," + " \"peers\": [" + " {" + " \"name\": \"my-server\"," + " \"url\": \"https://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"overwrite\"," + " \"trust-anchor\": \"!CA!\"," + " \"cert-file\": \"!CA!/kea-server.crt\"," + " \"key-file\": \"!CA!/kea-server.key\"," + " \"url\": \"https://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"disable\"," + " \"trust-anchor\": \"\"," + " \"cert-file\": \"\"," + " \"key-file\": \"\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"backup\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW(impl->configure(Element::fromJSON(patched))); + + // Check the global parameters. + std::string expected; + EXPECT_FALSE(impl->getConfig()->getTrustAnchor().unspecified()); + expected = TEST_CA_DIR; + expected += "/kea-ca.crt"; + EXPECT_EQ(expected, impl->getConfig()->getTrustAnchor().get()); + EXPECT_FALSE(impl->getConfig()->getCertFile().unspecified()); + expected = TEST_CA_DIR; + expected += "/kea-client.crt"; + EXPECT_EQ(expected, impl->getConfig()->getCertFile().get()); + EXPECT_FALSE(impl->getConfig()->getKeyFile().unspecified()); + expected = TEST_CA_DIR; + expected += "/kea-client.key"; + EXPECT_EQ(expected, impl->getConfig()->getKeyFile().get()); + EXPECT_FALSE(impl->getConfig()->getRequireClientCerts()); + EXPECT_TRUE(impl->getConfig()->getRestrictCommands()); + + // Check the first peer parameters: it inherits them from the global level. + HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig(); + ASSERT_TRUE(cfg); + EXPECT_TRUE(cfg->getTlsContext()); + + // Check the second peer parameters: it overwrites them. + cfg = impl->getConfig()->getPeerConfig("overwrite"); + ASSERT_TRUE(cfg); + EXPECT_FALSE(cfg->getTrustAnchor().unspecified()); + expected = TEST_CA_DIR; + EXPECT_EQ(expected, cfg->getTrustAnchor().get()); + EXPECT_FALSE(cfg->getCertFile().unspecified()); + expected = TEST_CA_DIR; + expected += "/kea-server.crt"; + EXPECT_EQ(expected, cfg->getCertFile().get()); + EXPECT_FALSE(cfg->getKeyFile().unspecified()); + expected = TEST_CA_DIR; + expected += "/kea-server.key"; + EXPECT_EQ(expected, cfg->getKeyFile().get()); + EXPECT_TRUE(cfg->getTlsContext()); + + // Check the last peer parameters: it disables TLS by setting them to "". + cfg = impl->getConfig()->getPeerConfig("disable"); + ASSERT_TRUE(cfg); + EXPECT_FALSE(cfg->getTrustAnchor().unspecified()); + EXPECT_EQ("", cfg->getTrustAnchor().get()); + EXPECT_FALSE(cfg->getCertFile().unspecified()); + EXPECT_EQ("", cfg->getCertFile().get()); + EXPECT_FALSE(cfg->getKeyFile().unspecified()); + EXPECT_EQ("", cfg->getKeyFile().get()); + // The TLS context should be null. + EXPECT_FALSE(cfg->getTlsContext()); +} + +// Test that a missing trust-anchor in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, missingTrustAnchor) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"trust-anchor\": \"\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server2: "; + expected += "trust-anchor parameter is missing or empty: "; + expected += "all or none of TLS parameters must be set"; + testInvalidConfig(patched, expected); +} + +// Test that a missing cert-file in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, missingCertFile) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"cert-file\": \"\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server2: "; + expected += "cert-file parameter is missing or empty: "; + expected += "all or none of TLS parameters must be set"; + testInvalidConfig(patched, expected); +} + +// Test that a missing key-file in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, missingKeyFile) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"key-file\": \"\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server2: "; + expected += "key-file parameter is missing or empty: "; + expected += "all or none of TLS parameters must be set"; + testInvalidConfig(patched, expected); +} + +// Test that a bad trust-anchor in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, badTrustAnchor) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"/this-file-does-not-exist\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server1: "; + expected += "load of CA file '/this-file-does-not-exist' failed: "; + // Backend dependent. +#ifdef WITH_OPENSSL + expected += "No such file or directory"; +#else + expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist"; +#endif + testInvalidConfig(patched, expected); +} + +// Test that a bad cert-file in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, badCertFile) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"/this-file-does-not-exist\"," + " \"key-file\": \"!CA!/kea-client.key\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server1: "; + expected += "load of cert file '/this-file-does-not-exist' failed: "; + // Backend dependent. +#ifdef WITH_OPENSSL + expected += "No such file or directory"; +#else + expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist"; +#endif + testInvalidConfig(patched, expected); +} + +// Test that a bad key-file in the HTTPS parameter set raise an error. +TEST_F(HAConfigTest, badKeyFile) { + const std::string ha_config = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"/this-file-does-not-exist\"," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"auto-failover\": false" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }" + " ]" + " }" + "]"; + const std::string& patched = replaceInConfig(ha_config, "!CA!", + TEST_CA_DIR); + std::string expected = "bad TLS config for server server1: "; + expected += "load of private key file '/this-file-does-not-exist' failed: "; + // Backend dependent. +#ifdef WITH_OPENSSL + expected += "No such file or directory"; +#else + expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist"; +#endif + testInvalidConfig(patched, expected); +} +#endif // WITH_OPENSSL || WITH_BOTAN_BOOST + +// Test that conversion of the role names works correctly. +TEST_F(HAConfigTest, stringToRole) { + EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, + HAConfig::PeerConfig::stringToRole("primary")); + EXPECT_EQ(HAConfig::PeerConfig::SECONDARY, + HAConfig::PeerConfig::stringToRole("secondary")); + EXPECT_EQ(HAConfig::PeerConfig::STANDBY, + HAConfig::PeerConfig::stringToRole("standby")); + EXPECT_EQ(HAConfig::PeerConfig::BACKUP, + HAConfig::PeerConfig::stringToRole("backup")); + EXPECT_THROW(HAConfig::PeerConfig::stringToRole("unsupported"), + BadValue); +} + +// Test that role name is generated correctly. +TEST_F(HAConfigTest, roleToString) { + EXPECT_EQ("primary", + HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::PRIMARY)); + EXPECT_EQ("secondary", + HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::SECONDARY)); + EXPECT_EQ("standby", + HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::STANDBY)); + EXPECT_EQ("backup", + HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::BACKUP)); +} + +// Test that conversion of the HA mode names works correctly. +TEST_F(HAConfigTest, stringToHAMode) { + EXPECT_EQ(HAConfig::LOAD_BALANCING, HAConfig::stringToHAMode("load-balancing")); + EXPECT_EQ(HAConfig::HOT_STANDBY, HAConfig::stringToHAMode("hot-standby")); + EXPECT_EQ(HAConfig::PASSIVE_BACKUP, HAConfig::stringToHAMode("passive-backup")); +} + +// Test that HA mode name is generated correctly. +TEST_F(HAConfigTest, HAModeToString) { + EXPECT_EQ("load-balancing", HAConfig::HAModeToString(HAConfig::LOAD_BALANCING)); + EXPECT_EQ("hot-standby", HAConfig::HAModeToString(HAConfig::HOT_STANDBY)); + EXPECT_EQ("passive-backup", HAConfig::HAModeToString(HAConfig::PASSIVE_BACKUP)); +} + +// Test that conversion of the 'pause' value works correctly. +TEST_F(HAConfigTest, stringToPausing) { + EXPECT_EQ(STATE_PAUSE_ALWAYS, + HAConfig::StateConfig::stringToPausing("always")); + EXPECT_EQ(STATE_PAUSE_NEVER, + HAConfig::StateConfig::stringToPausing("never")); + EXPECT_EQ(STATE_PAUSE_ONCE, + HAConfig::StateConfig::stringToPausing("once")); +} + +// Test that pause parameter value is generated correctly. +TEST_F(HAConfigTest, pausingToString) { + EXPECT_EQ("always", + HAConfig::StateConfig::pausingToString(STATE_PAUSE_ALWAYS)); + EXPECT_EQ("never", + HAConfig::StateConfig::pausingToString(STATE_PAUSE_NEVER)); + EXPECT_EQ("once", + HAConfig::StateConfig::pausingToString(STATE_PAUSE_ONCE)); +} + +// Verifies permutations of HA+MT configuration. +TEST_F(HAConfigTest, multiThreadingPermutations) { + + // Structure describing a test scenario. + struct Scenario { + std::string desc_; // Description of the scenario. + std::string mt_json_; // multi-threading config to use. + bool dhcp_mt_enabled_; // True if DHCP multi-threading is enabled. + uint32_t dhcp_threads_; // Value of DHCP thread-pool-size. + bool exp_ha_mt_enabled_; // If HA+MT should be enabled + bool exp_listener_; // If HA+MT should use dedicated listener. + uint32_t exp_listener_threads_; // Expected number of listener threads. + uint32_t exp_client_threads_; // Expected number of client threads. + }; + + // Mnemonic constants. + bool dhcp_mt = true; + bool ha_mt = true; + bool listener = true; + + std::vector scenarios { + { + "1 ha+mt by default", + "", + dhcp_mt, 4, + ha_mt, listener, 4, 4 + }, + { + "2 dhcp mt enabled, ha mt disabled", + makeHAMtJson(!ha_mt, !listener, 0, 0), + dhcp_mt, 4, + !ha_mt, !listener, 0, 0 + }, + { + "3 dhcp mt disabled, ha mt enabled", + makeHAMtJson(ha_mt, listener, 0, 0), + !dhcp_mt, 4, + !ha_mt, !listener, 0, 0 + }, + { + "4 dhcp mt enabled, ha mt enabled, listener disabled", + makeHAMtJson(ha_mt, !listener, 0, 0), + dhcp_mt, 4, + ha_mt, !listener, 4, 4 + }, + { + "5 dhcp mt enabled, ha mt enabled, listener enabled", + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 4, + ha_mt, listener, 4, 4 + }, + { + "6 explicit DHCP threads, explicit thread values", + makeHAMtJson(ha_mt, listener, 5, 6), + dhcp_mt, 4, + ha_mt, listener, 5, 6 + }, + { + "7 explicit DHCP threads, zero thread values", + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 8, + ha_mt, listener, 8, 8 + }, + { + "8 DHCP auto detect threads, zero thread values", + // Special case: if system reports supported threads as 0 + // then HA+MT should be disabled. Otherwise it should + // be enabled with listener and client threads set to the + // reported value. + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 0, + (hardware_threads_ > 0), listener, hardware_threads_, hardware_threads_ + } + }; + + // Iterate over the scenarios. + for (auto const& scenario : scenarios) { + SCOPED_TRACE(scenario.desc_); + + // Build the HA JSON configuration. + std::stringstream ss; + ss << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"wait-backup-ack\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"backup\"" + " }" + " ]"; + + if (!scenario.mt_json_.empty()) { + ss << "," << scenario.mt_json_; + } + + ss << "}]"; + ConstElementPtr config_json; + ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str())); + + // Set DHCP multi-threading configuration in CfgMgr. + setDHCPMultiThreadingConfig(scenario.dhcp_mt_enabled_, scenario.dhcp_threads_); + + // Create and configure the implementation. + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW_LOG(impl->configure(config_json)); + + // Fetch the updated config. + HAConfigPtr ha_config = impl->getConfig(); + + // Verify the configuration is as expected. + if (!scenario.exp_ha_mt_enabled_) { + // When HA+MT is disabled, the other values are moot. + ASSERT_FALSE(ha_config->getEnableMultiThreading()); + } else { + ASSERT_TRUE(ha_config->getEnableMultiThreading()); + EXPECT_EQ(ha_config->getHttpDedicatedListener(), scenario.exp_listener_); + EXPECT_EQ(ha_config->getHttpListenerThreads(), scenario.exp_listener_threads_); + EXPECT_EQ(ha_config->getHttpClientThreads(), scenario.exp_client_threads_); + } + } +} + +// Check that an IPv6 address can be used as part of a value for "url". +TEST_F(HAConfigTest, ipv6Url) { + std::string const ha_config(R"( + [ + { + "mode": "load-balancing", + "peers": [ + { + "name": "server1", + "role": "primary", + "url": "http://[2001:db8::1]:8080/" + }, + { + "name": "server2", + "role": "secondary", + "url": "http://[2001:db8::2]:8080/" + } + ], + "this-server-name": "server1" + } + ] + )"); + + // Configure HA. + HAImplPtr impl(new HAImpl()); + ASSERT_NO_THROW_LOG(impl->configure(Element::fromJSON(ha_config))); + + // Check the URL. + EXPECT_EQ(impl->getConfig()->getThisServerConfig()->getUrl().toText(), "http://[2001:db8::1]:8080/"); +} + +} // namespace diff --git a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc new file mode 100644 index 0000000..9cfe46c --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc @@ -0,0 +1,814 @@ +// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isc::asiolink; +using namespace isc::config; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::ha; +using namespace isc::ha::test; +using namespace isc::hooks; + +namespace { + +/// @brief Structure that holds registered hook indexes. +/// +/// This allows us to park packets. +struct TestHooks { + /// @brief Index of leases4_committed callout. + int hook_index_leases4_committed_; + + /// @brief Index of leases6_committed callout. + int hook_index_leases6_committed_; + + /// @brief Constructor + /// + /// The constructor registers hook points for callout tests. + TestHooks() { + hook_index_leases4_committed_ = + HooksManager::registerHook("leases4_committed"); + hook_index_leases6_committed_ = + HooksManager::registerHook("leases6_committed"); + } +}; + +TestHooks test_hooks; + +/// @brief Derivation of the @c HAImpl which provides access to protected +/// methods and members. +class TestHAImpl : public HAImpl { +public: + + using HAImpl::config_; + using HAImpl::service_; +}; + +/// @brief Test fixture class for @c HAImpl. +class HAImplTest : public HATest { +public: + + /// @brief Tests handler of a ha-sync command. + /// + /// It always expects that the error result is returned. The expected + /// error text should be provided as function argument. + /// + /// @param ha_sync_command command provided as text. + /// @param expected_response expected text response. + void testSynchronizeHandler(const std::string& ha_sync_command, + const std::string& expected_response) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + ConstElementPtr command = Element::fromJSON(ha_sync_command); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl.synchronizeHandler(*callout_handle)); + + ConstElementPtr response; + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_ERROR, expected_response); + } +}; + +// Tests that HAService object is created for DHCPv4 service. +TEST_F(HAImplTest, startService) { + // Valid configuration must be provided prior to starting the service. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Network state is also required. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + + // Start the service for DHCPv4 server. + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + // Make sure that the HA service has been created for the requested + // server type. + ASSERT_TRUE(ha_impl.service_); + EXPECT_EQ(HAServerType::DHCPv4, ha_impl.service_->getServerType()); +} + +// Tests that HAService object is created for DHCPv6 service. +TEST_F(HAImplTest, startService6) { + // Valid configuration must be provided prior to starting the service. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Network state is also required. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6)); + + // Start the service for DHCPv4 server. + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv6)); + + // Make sure that the HA service has been created for the requested + // server type. + ASSERT_TRUE(ha_impl.service_); + EXPECT_EQ(HAServerType::DHCPv6, ha_impl.service_->getServerType()); +} + +// Tests for buffer4_receive callout implementation. +TEST_F(HAImplTest, buffer4Receive) { + // Use hot-standby mode to make sure that this server instance is selected + // to process each received query. This is going to give predictable results. + ConstElementPtr ha_config = createValidJsonConfiguration(HAConfig::HOT_STANDBY); + + // Create implementation object and configure it. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(ha_config)); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + // Initially the HA service is in the waiting state and serves no scopes. + // We need to explicitly enable the scope to be served. + ha_impl.service_->serveDefaultScopes(); + + // Create callout handle to be used for passing arguments to the + // callout. + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + ASSERT_TRUE(callout_handle); + + // Create the BOOTP message. We can use it for testing message parsing + // failure case because BOOTP is not supported. We will later turn it + // into the DHCP message to test successful parsing. + std::vector msg = { + 1, // BOOTREQUEST + 1, // ethernet + 6, // HW address length = 6 + 0, // hops = 0 + 1, 2, 3, 4, // xid + 0, 0, // secs = 0 + 0, 0, // flags + 0, 0, 0, 0, // ciaddr = 0 + 0, 0, 0, 0, // yiaddr = 0 + 0, 0, 0, 0, // siaddr = 0 + 0, 0, 0, 0, // giaddr = 0 + 1, 2, 3, 4, 5, 6, // chaddr + }; + + // fill chaddr reminder, sname and file with zeros + msg.insert(msg.end(), 10 + 128 + 64, 0); + + // Create DHCPv4 message object from the BOOTP message. This should be + // successful because the message is not parsed yet. + Pkt4Ptr query4(new Pkt4(&msg[0], msg.size())); + + // Set buffer4_receive callout arguments. + callout_handle->setArgument("query4", query4); + + // Invoke the buffer4_receive callout. + ASSERT_NO_THROW(ha_impl.buffer4Receive(*callout_handle)); + + // The BOOTP messages are not supported so trying to unpack the message + // should trigger an error. The callout should set the next step to + // DROP treating the message as malformed. + EXPECT_EQ(CalloutHandle::NEXT_STEP_DROP, callout_handle->getStatus()); + // Malformed message should not be classified. + EXPECT_TRUE(query4->getClasses().empty()); + + // Turn this into the DHCP message by appending a magic cookie and the + // options. + std::vector magic_cookie = { + 99, 130, 83, 99 + }; + + // Provide DHCP message type option, truncated vendor option and domain name. + // Parsing this message should be successful but vendor option should be + // skipped. + std::vector options = { + 53, 1, 1, // Message type = DHCPDISCOVER + 125, 6, // vendor options + 1, 2, 3, 4, // enterprise id + 8, 1, // data len 8 but the actual length is 1 (truncated options) + 15, 3, 'a', 'b', 'c' // Domain name = abc + }; + + // Append the magic cookie and the options to our BOOTP message. + msg.insert(msg.end(), magic_cookie.begin(), magic_cookie.end()); + msg.insert(msg.end(), options.begin(), options.end()); + + // Create new query and pass it to the callout. + query4.reset(new Pkt4(&msg[0], msg.size())); + callout_handle->setArgument("query4", query4); + + // Invoke the callout again. + ASSERT_NO_THROW(ha_impl.buffer4Receive(*callout_handle)); + + // This time the callout should set the next step to SKIP to indicate to + // the DHCP server that the message has been already parsed. + EXPECT_EQ(CalloutHandle::NEXT_STEP_SKIP, callout_handle->getStatus()); + + // The client class should be assigned to the message to indicate that the + // server1 should process this message. + ASSERT_EQ(1, query4->getClasses().size()); + EXPECT_TRUE(query4->inClass("HA_server1")); + + // Check that the message has been parsed. The DHCP message type should + // be set in this case. + EXPECT_EQ(DHCPDISCOVER, static_cast(query4->getType())); + // Vendor option should be skipped because it was truncated. + EXPECT_FALSE(query4->getOption(DHO_VIVSO_SUBOPTIONS)); + // Domain name should not be skipped because the vendor option was truncated. + EXPECT_TRUE(query4->getOption(DHO_DOMAIN_NAME)); +} + +// Tests for buffer6_receive callout implementation. +TEST_F(HAImplTest, buffer6Receive) { + // Use hot-standby mode to make sure that this server instance is selected + // to process each received query. This is going to give predictable results. + ConstElementPtr ha_config = createValidJsonConfiguration(HAConfig::HOT_STANDBY); + + // Create implementation object and configure it. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(ha_config)); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv6)); + + // Initially the HA service is in the waiting state and serves no scopes. + // We need to explicitly enable the scope to be served. + ha_impl.service_->serveDefaultScopes(); + + // Create callout handle to be used for passing arguments to the + // callout. + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + ASSERT_TRUE(callout_handle); + + // Create DHCPv6 message. It initially has no transaction id so should be + // considered malformed. + std::vector msg = { + 1, // Solicit + }; + + // Create DHCPv4 message object from the BOOTP message. This should be + // successful because the message is not parsed yet. + Pkt6Ptr query6(new Pkt6(&msg[0], msg.size())); + + // Set buffer6_receive callout arguments. + callout_handle->setArgument("query6", query6); + + // Invoke the buffer6_receive callout. + ASSERT_NO_THROW(ha_impl.buffer6Receive(*callout_handle)); + + // Our DHCP messages contains no transaction id so it should cause + // parsing error. The next step is set to DROP for malformed messages. + EXPECT_EQ(CalloutHandle::NEXT_STEP_DROP, callout_handle->getStatus()); + // Malformed message should not be classified. + EXPECT_TRUE(query6->getClasses().empty()); + + // Append transaction id (3 bytes, each set to 1). + msg.insert(msg.end(), 3, 1); + + // Include 3 options in the DHCPv6 message: ORO, truncated vendor option + // and the NIS Domain Name option. This should be parsed correctly but the + // last option should be skipped because of the preceding option being + // truncated. + std::vector options = { + 0, 6, 0, 2, 0, 29, // option ORO requesting option 29 + 0, 17, // vendor options + 0, 9, // option length = 10 + 1, 2, 3, 4, // enterprise id + 0, 1, 0, 10, // code 1, (invalid) length = 10 + 1, // ONLY 1 byte of data (truncated) + 0, 29, 0, 3, 'a', 'b', 'c' // NIS Domain Name = abc + }; + + msg.insert(msg.end(), options.begin(), options.end()); + + // Create new query and pass it to the callout. + query6.reset(new Pkt6(&msg[0], msg.size())); + callout_handle->setArgument("query6", query6); + + // Invoke the callout again. + ASSERT_NO_THROW(ha_impl.buffer6Receive(*callout_handle)); + + // This time the callout should set the next step to SKIP to indicate to + // the DHCP server that the message has been already parsed. + EXPECT_EQ(CalloutHandle::NEXT_STEP_SKIP, callout_handle->getStatus()); + + // The client class should be assigned to the message to indicate that the + // server1 should process this message. + ASSERT_EQ(1, query6->getClasses().size()); + EXPECT_TRUE(query6->inClass("HA_server1")); + + // Check that the message has been parsed. The DHCP message type should + // be set in this case. + EXPECT_EQ(DHCPV6_SOLICIT, static_cast(query6->getType())); + // Domain name should be skipped because the vendor option was truncated. + EXPECT_FALSE(query6->getOption(D6O_NIS_DOMAIN_NAME)); +} + +// Tests leases4_committed callout implementation. +TEST_F(HAImplTest, leases4Committed) { + // Create implementation object and configure it. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + // Make sure we wait for the acks from the backup server to be able to + // test the case of sending lease updates even though the service is + // in the state in which the lease updates are normally not sent. + ha_impl.config_->setWaitBackupAck(true); + + // Create callout handle to be used for passing arguments to the + // callout. + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + ASSERT_TRUE(callout_handle); + + // Set the hook index so we can park packets. + callout_handle->setCurrentHook(test_hooks.hook_index_leases4_committed_); + + // query4 + Pkt4Ptr query4 = createMessage4(DHCPREQUEST, 1, 0, 0); + callout_handle->setArgument("query4", query4); + + // leases4 + Lease4CollectionPtr leases4(new Lease4Collection()); + callout_handle->setArgument("leases4", leases4); + + // deleted_leases4 + Lease4CollectionPtr deleted_leases4(new Lease4Collection()); + callout_handle->setArgument("deleted_leases4", deleted_leases4); + + // Set initial status. + callout_handle->setStatus(CalloutHandle::NEXT_STEP_CONTINUE); + + // Park the packet. + HooksManager::park("leases4_committed", query4, []{}); + + // There are no leases so the callout should return. + ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle)); + + // No updates are generated so the default status should not be modified. + EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4)); + + // Create a lease and pass it to the callout, but temporarily disable lease + // updates. + HWAddrPtr hwaddr(new HWAddr(std::vector(6, 1), HTYPE_ETHER)); + Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr, + static_cast(0), 0, + 60, 0, 1)); + leases4->push_back(lease4); + callout_handle->setArgument("leases4", leases4); + + ha_impl.config_->setSendLeaseUpdates(false); + + // Park the packet. + HooksManager::park("leases4_committed", query4, []{}); + + // Run the callout again. + ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle)); + + // No updates are generated so the default status should not be modified. + EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4)); + + // Enable updates and retry. + ha_impl.config_->setSendLeaseUpdates(true); + callout_handle->setArgument("leases4", leases4); + + // Park the packet. + HooksManager::park("leases4_committed", query4, []{}); + + // Run the callout again. + ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle)); + + // This time the lease update should be generated and the status should + // be set to "park". + EXPECT_EQ(CalloutHandle::NEXT_STEP_PARK, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4)); +} + +// Tests leases6_committed callout implementation. +TEST_F(HAImplTest, leases6Committed) { + // Create implementation object and configure it. + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv6)); + + // Make sure we wait for the acks from the backup server to be able to + // test the case of sending lease updates even though the service is + // in the state in which the lease updates are normally not sent. + ha_impl.config_->setWaitBackupAck(true); + + // Create callout handle to be used for passing arguments to the + // callout. + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + ASSERT_TRUE(callout_handle); + + // Set the hook index so we can park packets. + callout_handle->setCurrentHook(test_hooks.hook_index_leases6_committed_); + + // query6 + Pkt6Ptr query6 = createMessage6(DHCPV6_REQUEST, 1, 0); + callout_handle->setArgument("query6", query6); + + // leases6 + Lease6CollectionPtr leases6(new Lease6Collection()); + callout_handle->setArgument("leases6", leases6); + + // deleted_leases6 + Lease6CollectionPtr deleted_leases6(new Lease6Collection()); + callout_handle->setArgument("deleted_leases6", deleted_leases6); + + // Set initial status. + callout_handle->setStatus(CalloutHandle::NEXT_STEP_CONTINUE); + + // Park the packet. + HooksManager::park("leases6_committed", query6, []{}); + + // There are no leases so the callout should return. + ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle)); + + // No updates are generated so the default status should not be modified. + EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6)); + + // Create a lease and pass it to the callout, but temporarily disable lease + // updates. + DuidPtr duid(new DUID(std::vector(8, 2))); + Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"), duid, + 1234, 50, 60, 1)); + leases6->push_back(lease6); + callout_handle->setArgument("leases6", leases6); + + ha_impl.config_->setSendLeaseUpdates(false); + + // Park the packet. + HooksManager::park("leases6_committed", query6, []{}); + + // Run the callout again. + ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle)); + + // No updates are generated so the default status should not be modified. + EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6)); + + // Enable updates and retry. + ha_impl.config_->setSendLeaseUpdates(true); + callout_handle->setArgument("leases6", leases6); + + // Park the packet. + HooksManager::park("leases6_committed", query6, []{}); + + // Run the callout again. + ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle)); + + // This time the lease update should be generated and the status should + // be set to "park". + EXPECT_EQ(CalloutHandle::NEXT_STEP_PARK, callout_handle->getStatus()); + EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6)); +} + +// Tests ha-sync command handler with correct and incorrect arguments. +TEST_F(HAImplTest, synchronizeHandler) { + { + // This syntax is correct. The error returned is simply a result of + // trying to connect to the server which is offline, which should + // result in connection refused error. + SCOPED_TRACE("Correct syntax"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"," + " \"arguments\": {" + " \"server-name\": \"server2\"" + " }" + "}", "Connection refused"); + } + + { + SCOPED_TRACE("No arguments"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"" + "}", "arguments not found in the 'ha-sync' command"); + } + + { + SCOPED_TRACE("No server name"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"," + " \"arguments\": {" + " \"max-period\": 20" + " }" + "}", "'server-name' is mandatory for the 'ha-sync' command"); + } + + { + SCOPED_TRACE("Server name is not a string"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"," + " \"arguments\": {" + " \"server-name\": 20" + " }" + "}", "'server-name' must be a string in the 'ha-sync' command"); + } + + { + SCOPED_TRACE("Max period is not a number"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"," + " \"arguments\": {" + " \"server-name\": \"server2\"," + " \"max-period\": \"20\"" + " }" + "}", "'max-period' must be a positive integer in the 'ha-sync'" + " command"); + } + + { + SCOPED_TRACE("Max period must be positive"); + testSynchronizeHandler("{" + " \"command\": \"ha-sync\"," + " \"arguments\": {" + " \"server-name\": \"server2\"," + " \"max-period\": \"20\"" + " }" + "}", "'max-period' must be a positive integer in the 'ha-sync'" + " command"); + } + +} + +// Tests ha-continue command handler. +TEST_F(HAImplTest, continueHandler) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + ConstElementPtr command = Element::fromJSON("{ \"command\": \"ha-continue\" }"); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl.continueHandler(*callout_handle)); + + ConstElementPtr response; + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_SUCCESS, "HA state machine is not paused."); +} + +// Tests status-get command processed handler. +TEST_F(HAImplTest, statusGet) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + std::string name = "status-get"; + ConstElementPtr response = + Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }"); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + + callout_handle->setArgument("name", name); + callout_handle->setArgument("response", response); + + ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle)); + + ConstElementPtr got; + callout_handle->getArgument("response", got); + ASSERT_TRUE(got); + + std::string expected = + "{" + " \"arguments\": {" + " \"high-availability\": [" + " {" + " \"ha-mode\": \"load-balancing\"," + " \"ha-servers\": {" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ ]," + " \"state\": \"waiting\"" + " }," + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"," + " \"role\": \"secondary\"," + " \"communication-interrupted\": false," + " \"connecting-clients\": 0," + " \"unacked-clients\": 0," + " \"unacked-clients-left\": 0," + " \"analyzed-packets\": 0" + " }" + " }" + " }" + " ]," + " \"pid\": 1" + " }," + " \"result\": 0" + "}"; + EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected))); +} + +// Tests status-get command processed handler for backup server. +TEST_F(HAImplTest, statusGetBackupServer) { + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + ha_impl.config_->setThisServerName("server3"); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + std::string name = "status-get"; + ConstElementPtr response = + Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }"); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + + callout_handle->setArgument("name", name); + callout_handle->setArgument("response", response); + + ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle)); + + ConstElementPtr got; + callout_handle->getArgument("response", got); + ASSERT_TRUE(got); + + std::string expected = + "{" + " \"arguments\": {" + " \"high-availability\": [" + " {" + " \"ha-mode\": \"load-balancing\"," + " \"ha-servers\": {" + " \"local\": {" + " \"role\": \"backup\"," + " \"scopes\": [ ]," + " \"state\": \"backup\"" + " }" + " }" + " }" + " ]," + " \"pid\": 1" + " }," + " \"result\": 0" + "}"; + EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected))); +} + +// Tests status-get command processed handler for primary server being in the +// passive-backup state. +TEST_F(HAImplTest, statusGetPassiveBackup) { + TestHAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidPassiveBackupJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + std::string name = "status-get"; + ConstElementPtr response = + Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }"); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + + callout_handle->setArgument("name", name); + callout_handle->setArgument("response", response); + + ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle)); + + ConstElementPtr got; + callout_handle->getArgument("response", got); + ASSERT_TRUE(got); + + std::string expected = + "{" + " \"arguments\": {" + " \"high-availability\": [" + " {" + " \"ha-mode\": \"passive-backup\"," + " \"ha-servers\": {" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ \"server1\" ]," + " \"state\": \"passive-backup\"" + " }" + " }" + " }" + " ]," + " \"pid\": 1" + " }," + " \"result\": 0" + "}"; + EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected))); +} + +// Test ha-maintenance-notify command handler. +TEST_F(HAImplTest, maintenanceNotify) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + ConstElementPtr command = Element::fromJSON( + "{" + " \"command\": \"ha-maintenance-notify\"," + " \"arguments\": {" + " \"cancel\": false" + " }" + "}" + ); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl.maintenanceNotifyHandler(*callout_handle)); + + ConstElementPtr response; + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."); +} + +// Test ha-reset command handler. +TEST_F(HAImplTest, haReset) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + ConstElementPtr command = Element::fromJSON( + "{" + " \"command\": \"ha-reset\"" + "}" + ); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl.haResetHandler(*callout_handle)); + + ConstElementPtr response; + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."); +} + +} diff --git a/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc new file mode 100644 index 0000000..8aac8b0 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc @@ -0,0 +1,561 @@ +// Copyright (C) 2021-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + +using namespace isc::asiolink; +using namespace isc::config; +using namespace isc::data; +using namespace isc::dhcp; + +using namespace isc::ha; +using namespace isc::ha::test; +using namespace isc::http; +using namespace isc::util; + +namespace { + +/// @brief Derivation of the @c HAService which provides access to +/// protected methods and members. +class TestHAService : public HAService { +public: + + /// @brief Constructor. + /// + /// @param io_service Pointer to the IO service used by the DHCP server. + /// @param network_state Object holding state of the DHCP service + /// (enabled/disabled). + /// @param config Parsed HA hook library configuration. + /// @param server_type Server type, i.e. DHCPv4 or DHCPv6 server. + TestHAService(const IOServicePtr& io_service, + const NetworkStatePtr& network_state, + const HAConfigPtr& config, + const HAServerType& server_type = HAServerType::DHCPv4) + : HAService(io_service, network_state, config, server_type) { + } + + /// @brief Test version of the @c HAService::runModel. + /// + /// The original implementation of this method returns control when + /// @c NOP_EVT is found. This implementation runs a + /// single handler to allow the tests to verify if the state machine + /// transitions to an expected state before it is run again. + virtual void runModel(unsigned int event) { + try { + postNextEvent(event); + getState(getCurrState())->run(); + + } catch (const std::exception& ex) { + abortModel(ex.what()); + } + } + + /// @brief Schedules asynchronous "dhcp-disable" command to the specified + /// server. + /// + /// This variant of the method uses default HTTP client for communication. + /// + /// @param server_name name of the server to which the command should be + /// sent. + /// @param max_period maximum number of seconds for which the DHCP service + /// should be disabled. + /// @param post_request_action pointer to the function to be executed when + /// the request is completed. + void asyncDisableDHCPService(const std::string& server_name, + const unsigned int max_period, + const PostRequestCallback& post_request_action) { + HAService::asyncDisableDHCPService(*client_, server_name, max_period, + post_request_action); + } + + /// @brief Schedules asynchronous "dhcp-enable" command to the specified + /// server. + /// + /// This variant of the method uses default HTTP client for communication. + /// + /// @param server_name name of the server to which the command should be + /// sent. + /// @param post_request_action pointer to the function to be executed when + /// the request is completed. + void asyncEnableDHCPService(const std::string& server_name, + const PostRequestCallback& post_request_action) { + HAService::asyncEnableDHCPService(*client_, server_name, post_request_action); + } + + using HAService::asyncSendHeartbeat; + using HAService::asyncSyncLeases; + using HAService::postNextEvent; + using HAService::transition; + using HAService::verboseTransition; + using HAService::shouldSendLeaseUpdates; + using HAService::shouldQueueLeaseUpdates; + using HAService::pendingRequestSize; + using HAService::getPendingRequest; + using HAService::network_state_; + using HAService::config_; + using HAService::communication_state_; + using HAService::query_filter_; + using HAService::lease_update_backlog_; + using HAService::client_; + using HAService::listener_; +}; + +/// @brief Pointer to the @c TestHAService. +typedef boost::shared_ptr TestHAServicePtr; + +/// @brief Test fixture class for @c HAService multi-threading. +class HAMtServiceTest : public HATest { +public: + + /// @brief Constructor. + HAMtServiceTest() + : HATest() { + MultiThreadingMgr::instance().setMode(true); + CmdResponseCreator::command_accept_list_.clear(); + } + + /// @brief Destructor. + /// + /// Stops all test servers. + ~HAMtServiceTest() { + io_service_->get_io_service().reset(); + io_service_->poll(); + MultiThreadingMgr::instance().setMode(false); + CmdResponseCreator::command_accept_list_.clear(); + } + + /// @brief Callback function invoke upon test timeout. + /// + /// It stops the IO service and reports test timeout. + /// + /// @param fail_on_timeout Specifies if test failure should be reported. + void timeoutHandler(const bool fail_on_timeout) { + if (fail_on_timeout) { + ADD_FAILURE() << "Timeout occurred while running the test!"; + } + io_service_->stop(); + } +}; + +// Verifies HA+MT start, pause, resume, and stop. Note +// that pause and resume are tested indirectly through +// entry and exit of a critical section. +TEST_F(HAMtServiceTest, multiThreadingBasics) { + + // Build the HA JSON configuration. + std::stringstream ss; + ss << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"wait-backup-ack\": true," + " \"restrict-commands\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"backup\"" + " }" + " ]"; + + // Enable MT, listener, and 3 threads for both client and listener. + ss << "," << makeHAMtJson(true, true, 3, 3) << "}]"; + ConstElementPtr config_json; + ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str())); + + // Enable DHCP multi-threading configuration in CfgMgr with 3 threads. + setDHCPMultiThreadingConfig(true, 3); + + // Create the HA configuration + HAConfigPtr ha_config(new HAConfig()); + HAConfigParser parser; + ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json)); + + // Instantiate the service. + TestHAServicePtr service; + ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_, + ha_config))); + // Multi-threading should be enabled. + ASSERT_TRUE(ha_config->getEnableMultiThreading()); + + // Command filtering is enabled. + EXPECT_FALSE(CmdResponseCreator::command_accept_list_.empty()); + + // Now we'll start, pause, resume and stop a few times. + for (int i = 0; i < 3; ++i) { + // Verify we're stopped. + // Client should exist but be stopped. + ASSERT_TRUE(service->client_); + ASSERT_TRUE(service->client_->isStopped()); + if (i == 0) { + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + } else { + EXPECT_TRUE(service->client_->getThreadIOService()->stopped()); + } + + // Listener should exist but be stopped. + ASSERT_TRUE(service->listener_); + ASSERT_TRUE(service->listener_->isStopped()); + EXPECT_FALSE(service->listener_->getThreadIOService()); + + // Start client and listener. + ASSERT_NO_THROW_LOG(service->startClientAndListener()); + + // Verify we've started. + // Client should be running. + ASSERT_TRUE(service->client_->isRunning()); + ASSERT_TRUE(service->client_->getThreadIOService()); + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + EXPECT_EQ(service->client_->getThreadPoolSize(), 3); + EXPECT_EQ(service->client_->getThreadCount(), 3); + + // Listener should be running. + ASSERT_TRUE(service->listener_->isRunning()); + ASSERT_TRUE(service->listener_->getThreadIOService()); + EXPECT_FALSE(service->listener_->getThreadIOService()->stopped()); + EXPECT_EQ(service->listener_->getThreadPoolSize(), 3); + EXPECT_EQ(service->listener_->getThreadCount(), 3); + + { + // Entering a critical section should pause both client + // and listener. + MultiThreadingCriticalSection cs; + + // Client should be paused. + ASSERT_TRUE(service->client_->isPaused()); + EXPECT_TRUE(service->client_->getThreadIOService()->stopped()); + + // Listener should be paused. + ASSERT_TRUE(service->listener_->isPaused()); + EXPECT_TRUE(service->listener_->getThreadIOService()->stopped()); + } + + // Exiting critical section should resume both client + // and listener. + + // Client should be running. + ASSERT_TRUE(service->client_->isRunning()); + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + + // Listener should be running. + ASSERT_TRUE(service->listener_->isRunning()); + EXPECT_FALSE(service->listener_->getThreadIOService()->stopped()); + + // Stop should succeed. + ASSERT_NO_THROW_LOG(service->stopClientAndListener()); + } +} + +// Verifies multiThreadingBasics can be extended to use HTTPS/TLS> +TEST_F(HAMtServiceTest, multiThreadingTls) { + + // Build the HA JSON configuration. + std::stringstream ss; + ss << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"wait-backup-ack\": true," + " \"require-client-certs\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"https://127.0.0.1:8080/\"," + " \"role\": \"primary\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-server.crt\"," + " \"key-file\": \"!CA!/kea-server.key\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"https://127.0.0.1:8081/\"," + " \"role\": \"backup\"," + " \"trust-anchor\": \"!CA!/kea-ca.crt\"," + " \"cert-file\": \"!CA!/kea-client.crt\"," + " \"key-file\": \"!CA!/kea-client.key\"" + " }" + " ]"; + + // Enable MT, listener, and 3 threads for both client and listener. + ss << "," << makeHAMtJson(true, true, 3, 3) << "}]"; + ConstElementPtr config_json; + const std::string& patched = replaceInConfig(ss.str(), "!CA!", + TEST_CA_DIR); + ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(patched)); + + // Enable DHCP multi-threading configuration in CfgMgr with 3 threads. + setDHCPMultiThreadingConfig(true, 3); + + // Create the HA configuration + HAConfigPtr ha_config(new HAConfig()); + HAConfigParser parser; + ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json)); + + // Instantiate the service. + TestHAServicePtr service; + ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_, + ha_config))); + // Multi-threading should be enabled. + ASSERT_TRUE(ha_config->getEnableMultiThreading()); + + // Now we'll start, pause, resume and stop a few times. + for (int i = 0; i < 3; ++i) { + // Verify we're stopped. + // Client should exist but be stopped. + ASSERT_TRUE(service->client_); + ASSERT_TRUE(service->client_->isStopped()); + if (i == 0) { + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + } else { + EXPECT_TRUE(service->client_->getThreadIOService()->stopped()); + } + + // Listener should exist but be stopped. + ASSERT_TRUE(service->listener_); + ASSERT_TRUE(service->listener_->isStopped()); + EXPECT_FALSE(service->listener_->getThreadIOService()); + + // Start client and listener. + ASSERT_NO_THROW_LOG(service->startClientAndListener()); + + // Verify we've started. + // Client should be running. + ASSERT_TRUE(service->client_->isRunning()); + ASSERT_TRUE(service->client_->getThreadIOService()); + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + EXPECT_EQ(service->client_->getThreadPoolSize(), 3); + EXPECT_EQ(service->client_->getThreadCount(), 3); + + // Listener should be running. + ASSERT_TRUE(service->listener_->isRunning()); + ASSERT_TRUE(service->listener_->getThreadIOService()); + EXPECT_FALSE(service->listener_->getThreadIOService()->stopped()); + EXPECT_EQ(service->listener_->getThreadPoolSize(), 3); + EXPECT_EQ(service->listener_->getThreadCount(), 3); + + { + // Entering a critical section should pause both client + // and listener. + MultiThreadingCriticalSection cs; + + // Client should be paused. + ASSERT_TRUE(service->client_->isPaused()); + EXPECT_TRUE(service->client_->getThreadIOService()->stopped()); + + // Listener should be paused. + ASSERT_TRUE(service->listener_->isPaused()); + EXPECT_TRUE(service->listener_->getThreadIOService()->stopped()); + } + + // Exiting critical section should resume both client + // and listener. + + // Client should be running. + ASSERT_TRUE(service->client_->isRunning()); + EXPECT_FALSE(service->client_->getThreadIOService()->stopped()); + + // Listener should be running. + ASSERT_TRUE(service->listener_->isRunning()); + EXPECT_FALSE(service->listener_->getThreadIOService()->stopped()); + + // Stop should succeed. + ASSERT_NO_THROW_LOG(service->stopClientAndListener()); + } +} + +// Verifies permutations of HA+MT configuration and start-up. +TEST_F(HAMtServiceTest, multiThreadingConfigStartup) { + + // Structure describing a test scenario. + struct Scenario { + std::string desc_; // Description of the scenario. + std::string mt_json_; // multi-threading config to use. + bool dhcp_mt_enabled_; // True if DHCP multi-threading is enabled. + uint32_t dhcp_threads_; // Value of DHCP thread-pool-size. + bool exp_ha_mt_enabled_; // If HA+MT should be enabled + bool exp_listener_; // If HA+MT should use dedicated listener. + uint32_t exp_listener_threads_; // Expected number of listener threads. + uint32_t exp_client_threads_; // Expected number of client threads. + }; + + // Mnemonic constants. + bool dhcp_mt = true; + bool ha_mt = true; + bool listener = true; + + // Number of threads the system reports as supported. + uint32_t sys_threads = MultiThreadingMgr::detectThreadCount(); + + std::vector scenarios { + { + "1 ha+mt by default", + "", + dhcp_mt, 4, + ha_mt, listener, 4, 4 + }, + { + "2 dhcp mt enabled, ha mt disabled", + makeHAMtJson(!ha_mt, !listener, 0, 0), + dhcp_mt, 4, + !ha_mt, !listener, 0, 0 + }, + { + "3 dhcp mt disabled, ha mt enabled", + makeHAMtJson(ha_mt, listener, 0, 0), + !dhcp_mt, 4, + !ha_mt, !listener, 0, 0 + }, + { + "4 dhcp mt enabled, ha mt enabled, listener disabled", + makeHAMtJson(ha_mt, !listener, 0, 0), + dhcp_mt, 4, + ha_mt, !listener, 4, 4 + }, + { + "5 dhcp mt enabled, ha mt enabled, listener enabled", + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 4, + ha_mt, listener, 4, 4 + }, + { + "6 explicit DHCP threads, explicit thread values", + makeHAMtJson(ha_mt, listener, 5, 6), + dhcp_mt, 4, + ha_mt, listener, 5, 6 + }, + { + "7 explicit DHCP threads, zero thread values", + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 8, + ha_mt, listener, 8, 8 + }, + { + "8 DHCP auto detect threads, zero thread values", + // Special case: if system reports supported threads as 0 + // then HA+MT should be disabled. Otherwise it should + // be enabled with listener and client threads set to the + // reported value. + makeHAMtJson(ha_mt, listener, 0, 0), + dhcp_mt, 0, + (sys_threads > 0), listener, sys_threads, sys_threads + } + }; + + // Iterate over the scenarios. + for (auto const& scenario : scenarios) { + SCOPED_TRACE(scenario.desc_); + + // Build the HA JSON configuration. + std::stringstream ss; + ss << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"wait-backup-ack\": true," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:8080/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:8081/\"," + " \"role\": \"backup\"" + " }" + " ]"; + + if (!scenario.mt_json_.empty()) { + ss << "," << scenario.mt_json_; + } + + ss << "}]"; + ConstElementPtr config_json; + ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str())); + + // Set DHCP multi-threading configuration in CfgMgr. + setDHCPMultiThreadingConfig(scenario.dhcp_mt_enabled_, scenario.dhcp_threads_); + /// @todo this is a hack... we have chicken-egg... CmdHttpListener won't + /// start if MT is not enabled BUT that happens after config hook point + MultiThreadingMgr::instance().setMode(scenario.dhcp_mt_enabled_); + + // Create the HA configuration + HAConfigPtr ha_config(new HAConfig()); + HAConfigParser parser; + ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json)); + + // Instantiate the service. + TestHAServicePtr service; + ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_, + ha_config))); + ASSERT_NO_THROW_LOG(service->startClientAndListener()); + + // Verify the configuration is as expected. + if (!scenario.exp_ha_mt_enabled_) { + // When HA+MT is disabled, client should be single-threaded. + ASSERT_TRUE(service->client_); + EXPECT_FALSE(service->client_->getThreadIOService()); + EXPECT_EQ(service->client_->getThreadPoolSize(), 0); + EXPECT_EQ(service->client_->getThreadCount(), 0); + + // Listener should not exist. + ASSERT_FALSE(service->listener_); + continue; + } + + // Multi-threading should be enabled. + ASSERT_TRUE(ha_config->getEnableMultiThreading()); + + // When HA+MT is enabled, client should be multi-threaded. + ASSERT_TRUE(service->client_); + EXPECT_TRUE(service->client_->isRunning()); + EXPECT_TRUE(service->client_->getThreadIOService()); + EXPECT_EQ(service->client_->getThreadPoolSize(), scenario.exp_client_threads_); + + // Currently thread count should be the same as thread pool size. This might + // change if we go to so some sort of dynamic thread instance management. + EXPECT_EQ(service->client_->getThreadCount(), scenario.exp_client_threads_); + + if (!scenario.exp_listener_) { + // We should not have a listener. + ASSERT_FALSE(service->listener_); + continue; + } + + // We should have a running listener with the expected number of threads. + ASSERT_TRUE(service->listener_); + EXPECT_TRUE(service->listener_->isRunning()); + ASSERT_TRUE(service->listener_->getThreadIOService()); + EXPECT_EQ(service->listener_->getThreadPoolSize(), scenario.exp_listener_threads_); + + // Currently thread count should be the same as thread pool size. This might + // change if we go to so some sort of dynamic thread instance management. + EXPECT_EQ(service->listener_->getThreadCount(), scenario.exp_listener_threads_); + + ASSERT_NO_THROW_LOG(service->stopClientAndListener()); + } +} + +} // namespace diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc new file mode 100644 index 0000000..f0c286e --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc @@ -0,0 +1,8037 @@ +// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace isc::asiolink; +using namespace isc::config; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::ha; +using namespace isc::ha::test; +using namespace isc::hooks; +using namespace isc::http; +using namespace isc::util; + +/// @file The tests herein were created prior to HA+MT but are very valuable +/// for testing HA single-threaded operation and overall HA behavior. +/// HA+MT testing is done elsewhere. + +namespace { + +/// @brief IP address to which HTTP service is bound. +const std::string SERVER_ADDRESS = "127.0.0.1"; + +/// @brief Port number to which HTTP service is bound. +const unsigned short SERVER_PORT = 18123; + +/// @brief Request Timeout used in most of the tests (ms). +const long REQUEST_TIMEOUT = 10000; + +/// @brief Persistent connection idle timeout used in most of the tests (ms). +const long IDLE_TIMEOUT = 10000; + +/// @brief Test timeout (ms). +const long TEST_TIMEOUT = 10000; + +/// @brief Generates IPv4 leases to be used by the tests. +/// +/// @param [out] leases reference to the container where leases are stored. +void generateTestLeases(std::vector& leases) { + for (uint8_t i = 1; i <= 10; ++i) { + uint32_t lease_address = 0xC0000201 + 256 * i; + std::vector hwaddr(6, i); + Lease4Ptr lease(new Lease4(IOAddress(lease_address), + HWAddrPtr(new HWAddr(hwaddr, HTYPE_ETHER)), + ClientIdPtr(), + 60, + static_cast(1000 + i), + SubnetID(i))); + leases.push_back(lease); + } +} + +/// @brief Generates IPv6 leases to be used by the tests. +/// +/// @param [out] leases reference to the container where leases are stored. +void generateTestLeases(std::vector& leases) { + std::vector address_bytes = IOAddress("2001:db8:1::1").toBytes(); + for (uint8_t i = 1; i <= 10; ++i) { + DuidPtr duid(new DUID(std::vector(10, i))); + address_bytes[6] += i; + Lease6Ptr lease(new Lease6(Lease::TYPE_NA, + IOAddress::fromBytes(AF_INET6, &address_bytes[0]), + duid, 1, 50, 60, SubnetID(i))); + leases.push_back(lease); + } +} + +/// @brief Returns generated leases in JSON format. +/// +/// @tparam LeasesVec vector of IPv4 or IPv6 lease pointers. +/// @param leases reference to the container holding leases to be +/// converted to JSON format. +template +ConstElementPtr getLeasesAsJson(const LeasesVec& leases) { + ElementPtr leases_json = Element::createList(); + for (auto l = leases.begin(); l != leases.end(); ++l) { + leases_json->add((*l)->toElement()); + } + return (leases_json); +} + +/// @brief Derivation of the @c HAService which provides access to +/// protected methods and members. +class TestHAService : public HAService { +public: + + /// @brief Constructor. + /// + /// @param io_service Pointer to the IO service used by the DHCP server. + /// @param network_state Object holding state of the DHCP service + /// (enabled/disabled). + /// @param config Parsed HA hook library configuration. + /// @param server_type Server type, i.e. DHCPv4 or DHCPv6 server. + TestHAService(const IOServicePtr& io_service, + const NetworkStatePtr& network_state, + const HAConfigPtr& config, + const HAServerType& server_type = HAServerType::DHCPv4) + : HAService(io_service, network_state, config, server_type) { + } + + /// @brief Test version of the @c HAService::runModel. + /// + /// The original implementation of this method returns control when + /// @c NOP_EVT is found. This implementation runs a + /// single handler to allow the tests to verify if the state machine + /// transitions to an expected state before it is run again. + virtual void runModel(unsigned int event) { + try { + postNextEvent(event); + getState(getCurrState())->run(); + + } catch (const std::exception& ex) { + abortModel(ex.what()); + } + } + + /// @brief Schedules asynchronous "dhcp-disable" command to the specified + /// server. + /// + /// This variant of the method uses default HTTP client for communication. + /// + /// @param server_name name of the server to which the command should be + /// sent. + /// @param max_period maximum number of seconds for which the DHCP service + /// should be disabled. + /// @param post_request_action pointer to the function to be executed when + /// the request is completed. + void asyncDisableDHCPService(const std::string& server_name, + const unsigned int max_period, + const PostRequestCallback& post_request_action) { + HAService::asyncDisableDHCPService(*client_, server_name, max_period, + post_request_action); + } + + /// @brief Schedules asynchronous "dhcp-enable" command to the specified + /// server. + /// + /// This variant of the method uses default HTTP client for communication. + /// + /// @param server_name name of the server to which the command should be + /// sent. + /// @param post_request_action pointer to the function to be executed when + /// the request is completed. + void asyncEnableDHCPService(const std::string& server_name, + const PostRequestCallback& post_request_action) { + HAService::asyncEnableDHCPService(*client_, server_name, post_request_action); + } + + using HAService::asyncSendHeartbeat; + using HAService::asyncSyncLeases; + using HAService::postNextEvent; + using HAService::transition; + using HAService::verboseTransition; + using HAService::shouldSendLeaseUpdates; + using HAService::shouldQueueLeaseUpdates; + using HAService::pendingRequestSize; + using HAService::getPendingRequest; + using HAService::network_state_; + using HAService::config_; + using HAService::communication_state_; + using HAService::query_filter_; + using HAService::lease_update_backlog_; + using HAService::client_; + using HAService::listener_; +}; + +/// @brief Pointer to the @c TestHAService. +typedef boost::shared_ptr TestHAServicePtr; + +/// @brief Test HTTP response creator. +/// +/// It records received requests and allows the tests to retrieve them +/// to verify that they include expected values. +class TestHttpResponseCreator : public HttpResponseCreator { +public: + + /// @brief Constructor. + TestHttpResponseCreator() : + requests_(), control_result_(CONTROL_RESULT_SUCCESS), + arguments_(), per_request_control_result_(), + per_request_arguments_(), request_index_(), basic_auth_() { + } + + /// @brief Removes all received requests. + void clearReceivedRequests() { + std::lock_guard lk(mutex_); + requests_.clear(); + } + + /// @brief Returns a vector of received requests. + std::vector getReceivedRequests() { + std::lock_guard lk(mutex_); + return (requests_); + } + + /// @brief Finds a received request which includes two strings. + /// + /// @param str1 First string which must be included in the request. + /// @param str2 Second string which must be included in the request. + /// @param str3 Third string which must be included in the request. + /// It is optional and defaults to empty string which means "do not + /// match". + /// + /// @return Pointer to the request found, or null pointer if there is + /// no such request. + PostHttpRequestJsonPtr + findRequest(const std::string& str1, const std::string& str2, + const std::string& str3 = "") { + std::lock_guard lk(mutex_); + for (auto r = requests_.begin(); r < requests_.end(); ++r) { + std::string request_as_string = (*r)->toString(); + if (request_as_string.find(str1) != std::string::npos) { + if (request_as_string.find(str2) != std::string::npos) { + if (str3.empty() || + (request_as_string.find(str3) != std::string::npos)) + return (*r); + } + } + } + + // Request not found. + return (PostHttpRequestJsonPtr()); + } + + /// @brief Sets control result to be included in the responses. + /// + /// @param control_result new control result value. + void setControlResult(const int control_result) { + control_result_ = control_result; + } + + /// @brief Sets control result to be returned for the particular command. + /// + /// @param command_name command name. + /// @param control_result new control result value. + void setControlResult(const std::string& command_name, + const int control_result) { + std::lock_guard lk(mutex_); + per_request_control_result_[command_name] = control_result; + } + + /// @brief Sets arguments to be included in the responses. + /// + /// @param arguments pointer to the arguments. + void setArguments(const ElementPtr& arguments) { + std::lock_guard lk(mutex_); + arguments_ = arguments; + } + + /// @brief Sets arguments to be included in the response to a particular + /// command. + /// + /// Some tests require that the returned arguments vary for the same + /// command sent multiple times. One example of such command is the + /// @c lease4-get-page command sent multiple times to fetch pages of + /// leases from the partner. This method facilitates such test scenario. + /// In order to set different arguments for consecutive requests this + /// method must be called multiple times. Each call results in adding + /// arguments to be returned when the command is issued multiple times. + /// + /// @param command_name command name. + /// @param arguments pointer to the arguments. + void setArguments(const std::string& command_name, + const ElementPtr& arguments) { + std::lock_guard lk(mutex_); + per_request_arguments_[command_name].push_back(isc::data::copy(arguments)); + // Create request index for this command if it doesn't exist. + if (request_index_.count(command_name) == 0) { + request_index_[command_name] = 0; + } + } + + /// @brief Create a new request. + /// + /// @return Pointer to the new instance of the @ref HttpRequest. + virtual HttpRequestPtr + createNewHttpRequest() const { + return (HttpRequestPtr(new PostHttpRequestJson())); + } + + /// @brief Get basic HTTP authentication credentials. + /// + /// @return Basic HTTP authentication credentials and user id map. + const BasicHttpAuthMap& getCredentials() const { + return (basic_auth_.getCredentialMap()); + } + + /// @brief Add basic HTTP authentication client. + /// + /// @param user The user id to authorize. + /// @param password The password. + void addBasicAuth(const std::string& user, const std::string& password) { + std::lock_guard lk(mutex_); + basic_auth_.add(user, "", password, ""); + } + +private: + + /// @brief Creates HTTP response. + /// + /// @param request Pointer to the HTTP request. + /// @return Pointer to the generated HTTP response. + virtual HttpResponsePtr + createStockHttpResponse(const HttpRequestPtr& request, + const HttpStatusCode& status_code) const { + // The request hasn't been finalized so the request object + // doesn't contain any information about the HTTP version number + // used. But, the context should have this data (assuming the + // HTTP version is parsed ok). + HttpVersion http_version(request->context()->http_version_major_, + request->context()->http_version_minor_); + // This will generate the response holding JSON content. + HttpResponseJsonPtr response(new HttpResponseJson(http_version, status_code)); + response->finalize(); + return (response); + } + + /// @brief Creates HTTP response. + /// + /// It records received request so it may be later validated by the tests. + /// The returned status code and arguments are set using @c setControlResult + /// and @c setArguments methods. The per-request control result and arguments + /// take precedence over global values. + /// + /// @param request Pointer to the HTTP request. + /// @return Pointer to the generated HTTP OK response. + virtual HttpResponsePtr + createDynamicHttpResponse(HttpRequestPtr request) { + std::unique_lock lk(mutex_); + // Check authentication. + const BasicHttpAuthMap& credentials = getCredentials(); + if (!credentials.empty()) { + bool authorized = false; + try { + std::string value = request->getHeaderValue("Authorization"); + if (value.size() < 8) { + isc_throw(isc::BadValue, "header content is too short"); + } + if (value.substr(0, 6) != "Basic ") { + isc_throw(isc::BadValue, "no basic authentication"); + } + value = value.substr(6); + if (credentials.count(value) != 0) { + authorized = true; + } + } catch (const std::exception&) { + authorized = false; + } + if (!authorized) { + return (createStockHttpResponse(request, + HttpStatusCode::UNAUTHORIZED)); + } + } + + // Request must always be JSON. + PostHttpRequestJsonPtr request_json = + boost::dynamic_pointer_cast(request); + + // Remember the request received. + requests_.push_back(request_json); + + // The request must always contain non-empty Host header. + bool invalid_host = false; + try { + auto host_hdr = request_json->getHeader("Host"); + if (host_hdr->getValue().empty()) { + invalid_host = true; + } + + } catch (...) { + // Host header does not exist. + invalid_host = true; + } + + // If invalid host then return Bad Request. + if (invalid_host) { + return (createStockHttpResponse(request, HttpStatusCode::BAD_REQUEST)); + } + + int control_result = -1; + ElementPtr arguments; + + // First, check if the request contains a body with a command. + // If so, we may include a specific error code and arguments in the + // response based on the command name. + ConstElementPtr body = request_json->getBodyAsJson(); + if (body && (body->getType() == Element::map)) { + ConstElementPtr command = body->get("command"); + if (command && (command->getType() == Element::string)) { + std::string command_name = command->stringValue(); + + // Check if there is specific error code to be returned for this + // command. + if (per_request_control_result_.count(command_name) > 0) { + control_result = per_request_control_result_[command_name]; + } + + // Check if there are specific arguments to be returned for this + // command. + if (per_request_arguments_.count(command_name) > 0) { + // For certain requests we may return different arguments for consecutive + // instances of the same command. The request_index_ tracks the current + // index of the arguments to be returned. + arguments = per_request_arguments_[command_name][request_index_[command_name]]; + + // If we have reached the last arguments do not increase the index. Simply + // continue returning the same arguments. + if (request_index_[command_name] + 1 < per_request_arguments_[command_name].size()) { + // Not the last arguments, increase the index. + ++request_index_[command_name]; + } + } + } + } + + HttpResponseJsonPtr response(new HttpResponseJson(request->getHttpVersion(), + HttpStatusCode::OK)); + // Body is a list of responses from multiple servers listed in "service" + // argument of the request. + ElementPtr response_body = Element::createList(); + + // No per-command control result specified, so include the global result. + if (control_result < 0) { + control_result = control_result_; + } + + // No per-command arguments specified, so include the global arguments. + if (!arguments) { + arguments = arguments_; + } + + // Insert current date-time if not statically provided. + if (arguments && !arguments->contains("date-time")) { + arguments->set("date-time", Element::create(HttpDateTime().rfc1123Format())); + } + + // Insert unsent-update-count if not present. + if (arguments && !arguments->contains("unsent-update-count")) { + arguments->set("unsent-update-count", Element::create(int64_t(0))); + } + + response_body->add(boost::const_pointer_cast + (createAnswer(control_result, "response returned", + arguments))); + response->setBodyAsJson(response_body); + response->finalize(); + return (response); + } + + /// @brief Holds received HTTP requests. + std::vector requests_; + + /// @brief Control result to be returned in the server responses. + int control_result_; + + /// @brief Arguments to be included in the responses. + ElementPtr arguments_; + + /// @brief Command specific control results. + std::map per_request_control_result_; + + /// @brief Command specific response arguments. + std::map > per_request_arguments_; + + /// @brief Index of the next request of the given type. + std::map request_index_; + + /// @brief Basic HTTP authentication configuration. + BasicHttpAuthConfig basic_auth_; + + /// @brief Mutex to protect member access. + std::mutex mutex_; +}; + +/// @brief Shared pointer to the @c TestHttpResponseCreator. +typedef boost::shared_ptr TestHttpResponseCreatorPtr; + +/// @brief Implementation of the test @ref HttpResponseCreatorFactory. +/// +/// This factory class creates @ref TestHttpResponseCreator instances. +class TestHttpResponseCreatorFactory : public HttpResponseCreatorFactory { +public: + + /// @brief Constructor. + /// + /// Initializes common HTTP response creator instance. + TestHttpResponseCreatorFactory() + : creator_(new TestHttpResponseCreator()) { + } + + /// @brief Creates @ref TestHttpResponseCreator instance. + virtual HttpResponseCreatorPtr create() const { + return (creator_); + } + + /// @brief Returns instance of the response creator constructed by this + /// factory. + TestHttpResponseCreatorPtr getResponseCreator() const { + return (boost::dynamic_pointer_cast(creator_)); + } + +private: + + /// @brief Pointer to the common HTTP response creator. + HttpResponseCreatorPtr creator_; +}; + +/// @brief Pointer to the @c TestHttpResponseCreatorFactory. +typedef boost::shared_ptr +TestHttpResponseCreatorFactoryPtr; + +/// @brief Test fixture class for @c HAService. +/// +/// It creates 3 HTTP listeners (servers) which are used in the unit tests. +class HAServiceTest : public HATest { +public: + + struct MyState { + explicit MyState(const int state) + : state_(state) { + } + int state_; + }; + + struct PartnerState { + explicit PartnerState(const int state) + : state_(state) { + } + int state_; + }; + + struct FinalState { + explicit FinalState(const int state) + : state_(state) { + } + int state_; + }; + + /// @brief Constructor. + HAServiceTest() + : HATest(), + service_(), + factory_(new TestHttpResponseCreatorFactory()), + factory2_(new TestHttpResponseCreatorFactory()), + factory3_(new TestHttpResponseCreatorFactory()), + listener_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS), + SERVER_PORT, TlsContextPtr(), factory_, + HttpListener::RequestTimeout(REQUEST_TIMEOUT), + HttpListener::IdleTimeout(IDLE_TIMEOUT))), + listener2_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS), + SERVER_PORT + 1, TlsContextPtr(), factory2_, + HttpListener::RequestTimeout(REQUEST_TIMEOUT), + HttpListener::IdleTimeout(IDLE_TIMEOUT))), + listener3_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS), + SERVER_PORT + 2, TlsContextPtr(), factory3_, + HttpListener::RequestTimeout(REQUEST_TIMEOUT), + HttpListener::IdleTimeout(IDLE_TIMEOUT))), + leases4_(), + leases6_(), + user1_(""), + password1_(""), + user2_(""), + password2_(""), + user3_(""), + password3_("") { + MultiThreadingMgr::instance().setMode(false); + } + + /// @brief Destructor. + /// + /// Stops all test servers. + ~HAServiceTest() { + listener_->stop(); + listener2_->stop(); + listener3_->stop(); + io_service_->get_io_service().reset(); + io_service_->poll(); + MultiThreadingMgr::instance().setMode(false); + } + + /// @brief Callback function invoke upon test timeout. + /// + /// It stops the IO service and reports test timeout. + /// + /// @param fail_on_timeout Specifies if test failure should be reported. + void timeoutHandler(const bool fail_on_timeout) { + if (fail_on_timeout) { + ADD_FAILURE() << "Timeout occurred while running the test!"; + } + io_service_->stop(); + } + + /// @brief Creates a TestHAService instance with HA+MT disabled. + /// + /// @param network_state Object holding state of the DHCP service + /// (enabled/disabled). + /// @param config Parsed HA hook library configuration. + /// @param server_type server type, i.e. DHCPv4 or DHCPv6. + void createSTService(const NetworkStatePtr& state, + const HAConfigPtr config, + const HAServerType& server_type = HAServerType::DHCPv4) { + + ASSERT_FALSE(config->getEnableMultiThreading()); + ASSERT_NO_THROW_LOG(service_.reset(new TestHAService(io_service_, state, + config, server_type))); + ASSERT_TRUE(service_->client_); + ASSERT_FALSE(service_->client_->getThreadIOService()); + ASSERT_FALSE(service_->listener_); + } + + /// @brief Generates IPv4 leases to be used by the tests. + void generateTestLeases4() { + generateTestLeases(leases4_); + } + + /// @brief Returns range of generated IPv4 leases in JSON format. + /// + /// @param first_index Index of the first lease to be returned. + /// @param last_index Index of the last lease to be returned. + ConstElementPtr getTestLeases4AsJson(const size_t first_index, + const size_t last_index) const { + return (getLeasesAsJson(std::vector(leases4_.begin() + first_index, + leases4_.begin() + last_index))); + } + + /// @brief Generates IPv6 leases to be used by the tests. + void generateTestLeases6() { + generateTestLeases(leases6_); + } + + /// @brief Returns range of generated IPv6 leases in JSON format. + /// + /// @param first_index Index of the first lease to be returned. + /// @param last_index Index of the last lease to be returned. + ConstElementPtr getTestLeases6AsJson(const size_t first_index, + const size_t last_index) const { + return (getLeasesAsJson(std::vector(leases6_.begin() + first_index, + leases6_.begin() + last_index))); + } + + /// @brief Configure server side to return IPv4 leases in 3-element + /// chunks. + /// + /// Leases are fetched in pages, so the lease4-get-page should be + /// sent multiple times. The server is configured to return leases + /// in 3-element chunks. Note that the HA configis set to ask for 3 + /// leases. + void createPagedSyncResponses4() { + ElementPtr response_arguments = Element::createMap(); + + // First, return leases with indexes from 0 to 2. + response_arguments->set("leases", getTestLeases4AsJson(0, 3)); + factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + + // Next, return leases with indexes from 3 to 5. + response_arguments->set("leases", getTestLeases4AsJson(3, 6)); + factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + + // Then, return leases with indexes from 6 to 8. + response_arguments->set("leases", getTestLeases4AsJson(6, 9)); + factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + + // Finally, there is one lease with index 9 to be returned. + // When the server requests a page of 3 leases and gets 1 it + // means that the last page was returned. At this point, the + // server ends synchronization. + response_arguments->set("leases", getTestLeases4AsJson(9, 10)); + factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + } + + /// @brief Configure server side to return IPv6 leases in 3-element + /// chunks. + /// + /// Leases are fetched in pages, so the lease6-get-page should be + /// sent multiple times. The server is configured to return leases + /// in 3-element chunks. Note that the HA configis set to ask for 3 + /// leases. + void createPagedSyncResponses6() { + ElementPtr response_arguments = Element::createMap(); + + // First, return leases with indexes from 0 to 2. + response_arguments->set("leases", getTestLeases6AsJson(0, 3)); + factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + + // Next, return leases with indexes from 3 to 5. + response_arguments->set("leases", getTestLeases6AsJson(3, 6)); + factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + + // Then, return leases with indexes from 6 to 8. + response_arguments->set("leases", getTestLeases6AsJson(6, 9)); + factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + + // Finally, there is one lease with index 9 to be returned. + // When the server requests a page of 3 leases and gets 1 it + // means that the last page was returned. At this point, the + // server ends synchronization. + response_arguments->set("leases", getTestLeases6AsJson(9, 10)); + factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments); + } + + /// @brief Set basic HTTP authentication in a config. + /// + /// @param config Configuration to update. + void setBasicAuth(HAConfigPtr config) { + if (!config) { + ADD_FAILURE() << "null config"; + return; + } + if (!user1_.empty()) { + HAConfig::PeerConfigPtr peer = config->getPeerConfig("server1"); + if (!peer) { + ADD_FAILURE() << "null server1 config"; + return; + } + BasicHttpAuthPtr& auth = peer->getBasicAuth(); + auth.reset(new BasicHttpAuth(user1_, password1_)); + } + if (!user2_.empty()) { + HAConfig::PeerConfigPtr peer = config->getPeerConfig("server2"); + if (!peer) { + ADD_FAILURE() << "null server2 config"; + return; + } + BasicHttpAuthPtr& auth = peer->getBasicAuth(); + auth.reset(new BasicHttpAuth(user2_, password2_)); + } + if (!user3_.empty()) { + HAConfig::PeerConfigPtr peer = config->getPeerConfig("server3"); + if (!peer) { + ADD_FAILURE() << "null server3 config"; + return; + } + BasicHttpAuthPtr& auth = peer->getBasicAuth(); + auth.reset(new BasicHttpAuth(user3_, password3_)); + } + } + + /// @brief Tests scenarios when lease updates are sent to a partner while + /// the partner is online or offline. + /// + /// @param unpark_handler a function called when packet is unparked. + /// @param should_fail indicates if the update is expected to be unsuccessful. + /// @param num_updates expected number of servers to which lease updates are + /// sent. + /// @param my_state state of the server while lease updates are sent. + /// @param wait_backup_ack indicates if the server should wait for the acknowledgment + /// from the backup servers. + /// @param create_service a boolean flag indicating whether the test should + /// re-create HA service and communication state. + void testSendLeaseUpdates(std::function unpark_handler, + const bool should_fail, + const size_t num_updates, + const MyState& my_state = MyState(HA_LOAD_BALANCING_ST), + const bool wait_backup_ack = false, + const bool create_service = true) { + // Create parking lot where query is going to be parked and unparked. + ParkingLotPtr parking_lot(new ParkingLot()); + ParkingLotHandlePtr parking_lot_handle(new ParkingLotHandle(parking_lot)); + + // Create query. + Pkt4Ptr query(new Pkt4(DHCPREQUEST, 1234)); + + // Create leases collection and put the lease there. + Lease4CollectionPtr leases4(new Lease4Collection()); + HWAddrPtr hwaddr(new HWAddr(std::vector(6, 1), HTYPE_ETHER)); + Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr, + static_cast(0), 0, + 60, 0, 1)); + leases4->push_back(lease4); + + // Create deleted leases collection and put the lease there too. + Lease4CollectionPtr deleted_leases4(new Lease4Collection()); + Lease4Ptr deleted_lease4(new Lease4(IOAddress("192.2.3.4"), hwaddr, + static_cast(0), 0, + 60, 0, 1)); + deleted_leases4->push_back(deleted_lease4); + + if (create_service) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + config_storage->setWaitBackupAck(wait_backup_ack); + // Let's override the default value. The lower value makes it easier + // for some unit tests to simulate the server's overflow. Simulating it + // requires appending leases to the backlog. It is easier to add 10 + // than 100. + config_storage->setDelayedUpdatesLimit(10); + setBasicAuth(config_storage); + + // The communication state is the member of the HAServce object. We have to + // replace this object with our own implementation to have an ability to + // modify its poke time. + NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_, + config_storage)); + // Set poke time 30s in the past. If the state is poked it will be reset + // to the current time. This allows for testing whether the object has been + // poked by the HA service. + state->modifyPokeTime(-30); + + // Create HA service. + createSTService(network_state_, config_storage); + service_->communication_state_ = state; + } + + service_->transition(my_state.state_, HAService::NOP_EVT); + + // Schedule lease updates. + EXPECT_EQ(num_updates, + service_->asyncSendLeaseUpdates(query, leases4, deleted_leases4, + parking_lot_handle)); + + // The number of pending requests should be 2 times the number of + // contacted servers because we send one lease update and one + // lease deletion to each contacted server from which we expect + // an acknowledgment. + EXPECT_EQ(2 * num_updates, service_->getPendingRequest(query)); + + // Let's park the packet and associate it with the callback function which + // simply records the fact that it has been called. We expect that it wasn't + // because the parked packet should be dropped as a result of lease updates + // failures. + ASSERT_NO_THROW(parking_lot->park(query, unpark_handler)); + + ASSERT_NO_THROW(parking_lot->reference(query)); + + // Actually perform the lease updates. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, [this]() { + // Finish running IO service when there are no more pending requests. + return (service_->pendingRequestSize() == 0); + })); + + // Only if we wait for lease updates to complete it makes sense to test + // that the packet was either dropped or unparked. + if (num_updates > 0) { + // Try to drop the packet. We expect that the packet has been already + // dropped so this should return false. + EXPECT_FALSE(parking_lot_handle->drop(query)); + } + + // The updates should not be sent to this server. + EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty()); + + if (should_fail) { + EXPECT_EQ(HA_UNAVAILABLE_ST, service_->communication_state_->getPartnerState()); + } else { + EXPECT_NE(service_->communication_state_->getPartnerState(), HA_UNAVAILABLE_ST); + } + } + + /// @brief Tests scenarios when IPv6 lease updates are sent to a partner while + /// the partner is online or offline. + /// + /// @param unpark_handler a function called when packet is unparked. + /// @param should_fail indicates if the update is expected to be unsuccessful. + /// @param num_updates expected number of servers to which lease updates are + /// sent. + /// @param my_state state of the server while lease updates are sent. + /// @param wait_backup_ack indicates if the server should wait for the acknowledgment + /// from the backup servers. + /// @param create_service a boolean flag indicating whether the test should + /// re-create HA service and communication state. + void testSendLeaseUpdates6(std::function unpark_handler, + const bool should_fail, + const size_t num_updates, + const MyState& my_state = MyState(HA_LOAD_BALANCING_ST), + const bool wait_backup_ack = false, + const bool create_service = true) { + + // Create parking lot where query is going to be parked and unparked. + ParkingLotPtr parking_lot(new ParkingLot()); + ParkingLotHandlePtr parking_lot_handle(new ParkingLotHandle(parking_lot)); + + DuidPtr duid(new DUID(std::vector(8, 2))); + + // Create query. + Pkt6Ptr query(new Pkt6(DHCPV6_SOLICIT, 1234)); + OptionPtr opt_client_id(new Option(Option::V6, + D6O_CLIENTID, + duid->getDuid().begin(), + duid->getDuid().end())); + query->addOption(opt_client_id); + + // Create leases collection and put the lease there. + Lease6CollectionPtr leases6(new Lease6Collection()); + Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"), duid, + 1234, 50, 60, 1)); + leases6->push_back(lease6); + + // Create deleted leases collection and put the lease there too. + Lease6CollectionPtr deleted_leases6(new Lease6Collection()); + Lease6Ptr deleted_lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::efac"), + duid, 1234, 50, 60, 1)); + deleted_leases6->push_back(deleted_lease6); + + if (create_service) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + config_storage->setDelayedUpdatesLimit(10); + config_storage->setWaitBackupAck(wait_backup_ack); + setBasicAuth(config_storage); + + // The communication state is the member of the HAServce object. We have to + // replace this object with our own implementation to have an ability to + // modify its poke time. + NakedCommunicationState6Ptr state(new NakedCommunicationState6(io_service_, + config_storage)); + // Set poke time 30s in the past. If the state is poked it will be reset + // to the current time. This allows for testing whether the object has been + // poked by the HA service. + state->modifyPokeTime(-30); + + // Create HA service. + createSTService(network_state_, config_storage, HAServerType::DHCPv6); + service_->communication_state_ = state; + } + + service_->transition(my_state.state_, HAService::NOP_EVT); + + // Schedule lease updates. + EXPECT_EQ(num_updates, + service_->asyncSendLeaseUpdates(query, leases6, deleted_leases6, + parking_lot_handle)); + + // The number of requests we send is equal to the number of servers + // from which we expect an acknowledgement. We send both lease updates + // and the deletions in a single bulk update command. + EXPECT_EQ(num_updates, service_->getPendingRequest(query)); + + EXPECT_FALSE(boost::dynamic_pointer_cast + (service_->communication_state_)->isPoked()); + + // Let's park the packet and associate it with the callback function which + // simply records the fact that it has been called. We expect that it wasn't + // because the parked packet should be dropped as a result of lease updates + // failures. + ASSERT_NO_THROW(parking_lot->park(query, unpark_handler)); + + ASSERT_NO_THROW(parking_lot->reference(query)); + + // Actually perform the lease updates. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, [this]() { + // Finish running IO service when there are no more pending requests. + return (service_->pendingRequestSize() == 0); + })); + + // Only if we wait for lease updates to complete it makes sense to test + // that the packet was either dropped or unparked. + if (num_updates > 0) { + // Try to drop the packet. We expect that the packet has been already + // dropped so this should return false. + EXPECT_FALSE(parking_lot_handle->drop(query)); + } + + // The updates should not be sent to this server. + EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty()); + + if (should_fail) { + EXPECT_EQ(HA_UNAVAILABLE_ST, service_->communication_state_->getPartnerState()); + } + } + + /// @brief Tests scenarios when recurring heartbeat has been enabled + /// and the partner is online or offline. + /// + /// @param control_result control result that the servers should return. + /// @param should_pass boolean value indicating if the heartbeat should + /// be successful or not. + void testRecurringHeartbeat(const int control_result, + const bool should_pass) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + config_storage->setHeartbeatDelay(1000); + setBasicAuth(config_storage); + + // Create a valid static response to the heartbeat command. + ElementPtr response_arguments = Element::createMap(); + response_arguments->set("state", Element::create(std::string("load-balancing"))); + + // Both server 2 and server 3 are configured to send this response. + factory2_->getResponseCreator()->setArguments(response_arguments); + factory3_->getResponseCreator()->setArguments(response_arguments); + + // Configure server 2 and server 3 to send a specified control result. + factory2_->getResponseCreator()->setControlResult(control_result); + factory3_->getResponseCreator()->setControlResult(control_result); + + // The communication state is the member of the HAServce object. We have to + // replace this object with our own implementation to have an ability to + // modify its poke time. + NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_, + config_storage)); + // Set poke time 30s in the past. If the state is poked it will be reset + // to the current time. This allows for testing whether the object has been + // poked by the HA service. + state->modifyPokeTime(-30); + + // Create the service and replace the default communication state object. + TestHAService service(io_service_, network_state_, config_storage); + service.communication_state_ = state; + + EXPECT_FALSE(state->isPoked()); + + // Let's explicitly transition the state machine to the load balancing state + // in which the periodic heartbeats should be generated. + ASSERT_NO_THROW(service.verboseTransition(HA_LOAD_BALANCING_ST)); + ASSERT_NO_THROW(service.runModel(HAService::NOP_EVT)); + + // Run the IO service to allow the heartbeat interval timers to execute. + ASSERT_NO_THROW(runIOService(2000)); + + // Server 1 and server 3 must never receive heartbeats because the former + // is the one that generates them and the latter is a backup server. + EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty()); + EXPECT_TRUE(factory3_->getResponseCreator()->getReceivedRequests().empty()); + + // If should pass, the communication state should be poked. + if (should_pass) { + EXPECT_TRUE(state->isPoked()); + } else { + EXPECT_FALSE(state->isPoked()); + } + } + + /// @brief Tests scenarios when all lease updates are sent successfully. + void testSendSuccessfulUpdates() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { unpark_called = true; }, + false, 1); + + // Expecting that the packet was unparked because lease + // updates are expected to be successful. + EXPECT_TRUE(unpark_called); + + // Updates have been sent so this counter should remain 0. + EXPECT_EQ(0, service_->communication_state_->getUnsentUpdateCount()); + + // The server 2 should have received two commands. + EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 2 has received lease4-update command. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request2); + + // Check that the server 2 has received lease4-del command. + auto delete_request2 = + factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request2); + + // Lease updates should be successfully sent to server3. + EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 3 has received lease4-update command. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request3); + + // Check that the server 3 has received lease4-del command. + auto delete_request3 = + factory3_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request3); + } + + /// @brief Tests that DHCPv4 lease updates are queued when the server is in the + /// communication-recovery state and later sent before transitioning back to + /// the load-balancing state. + void testSendUpdatesCommunicationRecovery() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST)); + + // Packet shouldn't be unparked because no updates went out. We merely + // queued the updates. + EXPECT_FALSE(unpark_called); + + // Let's make sure they have been queued. + EXPECT_EQ(2, service_->lease_update_backlog_.size()); + + // Make partner available. + service_->communication_state_->poke(); + service_->communication_state_->setPartnerState("load-balancing"); + + // This should cause the server to send outstanding lease updates and + // because they are all successful the server should transition to the + // load-balancing state and continue normal operation. + testSynchronousCommands([this]() { + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); + }); + + // Lease updates should have been sent. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4")); + + // Backlog should be empty. + EXPECT_EQ(0, service_->lease_update_backlog_.size()); + } + + /// @brief Test the cases when the trying to recover from the communication + /// interruption and sending lease updates or/and ha-reset fails. + /// + /// @param partner_state partner state when communication is re-established. + /// @param lease_update_result control result returned to lease updates. + /// @param ha_reset_result control result returned to ha-reset command. + /// @param overflow boolean value indicating if this test should verify the + /// case when the leases backlog is overflown (when true), or not (when + /// false). + void testSendUpdatesCommunicationRecoveryFailed(const std::string& partner_state, + const int lease_update_result, + const int ha_reset_result, + const bool overflow = false) { + // Partner responds with a specified control result to lease updates. + factory2_->getResponseCreator()->setControlResult("lease4-update", + lease_update_result); + factory2_->getResponseCreator()->setControlResult("lease4-del", + lease_update_result); + // Partner returns specified control result to ha-reset. + factory2_->getResponseCreator()->setControlResult("ha-reset", ha_reset_result); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST)); + + // Packet shouldn't be unparked because no updates went out. We merely + // queued the updates. + EXPECT_FALSE(unpark_called); + + // Let's make sure they have been queued. + EXPECT_EQ(2, service_->lease_update_backlog_.size()); + + // When testing the case when the backlog should be overflown, we need + // to add several more leases to the backlog to exceed the limit. + if (overflow) { + ASSERT_NO_THROW(generateTestLeases4()); + for (auto lease : leases4_) { + service_->lease_update_backlog_.push(LeaseUpdateBacklog::ADD, lease); + } + } + + // Make partner available. + service_->communication_state_->poke(); + service_->communication_state_->setPartnerState(partner_state); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This should cause the server to attempt to send outstanding lease + // updates to the partner. + testSynchronousCommands([this, ha_reset_result]() { + service_->runModel(HAService::NOP_EVT); + // If the ha-reset returns success the server should transition to the + // waiting state and begin synchronization. Otherwise, if the ha-reset + // fails the server should wait in the communication-recovery state + // until it succeeds. + if (ha_reset_result == CONTROL_RESULT_SUCCESS) { + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + } else { + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); + } + }); + + // The server will only send lease updates if it is not overflown. If + // it is overflown, it will rather transition to the waiting state to + // initiate full synchronization. + if (!overflow) { + // Deletions are scheduled first and this should cause the failure. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4")); + // This lease update should not be sent because the first update + // triggered an error. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3")); + } + + if ((partner_state == "load-balancing") || (partner_state == "communication-recovery")) { + // The lease updates failed and the partner remains in the load-balancing or + // communication-recovery state, so the server should send ha-reset to the + // partner to cause it to transition to the waiting state and synchronize + // the lease database. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-reset", "")); + } else { + // The lease updates failed but the partner is already synchronizing lease + // database. In this case, don't send the ha-reset. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-reset", "")); + } + + // The backlog should be empty. + EXPECT_EQ(0, service_->lease_update_backlog_.size()); + } + + /// @brief Tests scenarios when lease updates are not sent to the failover peer. + void testSendUpdatesPartnerDown() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_PARTNER_DOWN_ST)); + + // There were no lease updates for which we have been waiting + // to complete so the packet was never unparked. Note that in + // such situation the packet is not parked either. + EXPECT_FALSE(unpark_called); + + // In the partner-down state we don't send lease updates. We + // should count transactions for which lease updates were not sent. + // This is later returned in the heartbeat so the partner can + // determine whether it should synchronize its lease database or + // not. + EXPECT_EQ(1, service_->communication_state_->getUnsentUpdateCount()); + + // Server 2 should not receive lease4-update. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_FALSE(update_request2); + + // Server 2 should not receive lease4-del. + auto delete_request2 = + factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_FALSE(delete_request2); + } + + /// @brief Tests scenarios when one of the servers to which + /// updates are sent is offline. + void testSendUpdatesActiveServerOffline() { + // Start only two servers out of three. The server 3 is not running. + ASSERT_NO_THROW({ + listener_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // Server 2 should not receive lease4-update. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_FALSE(update_request2); + + // Server 2 should not receive lease4-del. + auto delete_request2 = + factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_FALSE(delete_request2); + } + + /// @brief Tests scenarios when one of the servers to which a + /// lease update is sent returns an error. + void testSendUpdatesControlResultError() { + // Instruct the server 2 to return an error as a result of receiving a command. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // The updates should be sent to server 2 and this server should + // return error code. + EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Server 2 should receive lease4-update. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request2); + + // Server 2 should receive lease4-del. + auto delete_request2 = + factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request2); + + // Lease updates should be successfully sent to server3. + EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 3 has received lease4-update command. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request3); + + // Check that the server 3 has received lease4-del command. + auto delete_request3 = + factory3_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request3); + } + + /// @brief Tests scenarios when one of the servers to which a + /// lease update is sent does not authorize the local server. + void testSendUpdatesControlResultUnauthorized() { + // Instruct the server 2 to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // The updates should be sent to server 2 and this server should + // return error code. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); + + // Lease updates should be successfully sent to server3. + EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 3 has received lease4-update command. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request3); + + // Check that the server 3 has received lease4-del command. + auto delete_request3 = + factory3_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request3); + } + + /// @brief Tests scenarios when one of the servers to which + /// updates are sent is offline. + void testSendUpdatesBackupServerOffline() { + // Start only two servers out of three. The server 2 is not running. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { unpark_called = true; }, + false, 1); + + EXPECT_TRUE(unpark_called); + + // The server 2 should have received two commands. + EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 2 has received lease4-update command. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_TRUE(update_request2); + + // Check that the server 2 has received lease4-del command. + auto delete_request2 = + factory2_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_TRUE(delete_request2); + + // Server 3 should not receive lease4-update. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease4-update", + "192.1.2.3"); + EXPECT_FALSE(update_request3); + + // Server 3 should not receive lease4-del. + auto delete_request3 = + factory3_->getResponseCreator()->findRequest("lease4-del", + "192.2.3.4"); + EXPECT_FALSE(delete_request3); + } + + /// @brief Test the scenario when the servers receiving a lease update + /// return the conflict status code. + void testSendUpdatesControlResultConflict() { + // Instruct the server 2 to return an error as a result of receiving a command. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT); + factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, false, 1); + + // Ensure that the server has recorded a lease update conflict. The conflict + // reported by the backup server should not count. + EXPECT_EQ(1, service_->communication_state_->getRejectedLeaseUpdatesCount()); + + // Change the partner's response to success. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_SUCCESS); + + // Try sending the lease updates again. The previously rejected lease should + // now be accepted and the counter should be 0. + bool unpark_called = false; + testSendLeaseUpdates([&unpark_called] { + unpark_called = true; + }, false, 1, MyState(HA_LOAD_BALANCING_ST), true, false); + EXPECT_TRUE(unpark_called); + EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount()); + } + + /// @brief Tests scenarios when all lease updates are sent successfully. + void testSendSuccessfulUpdates6() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 1); + + // Expecting that the packet was unparked because lease + // updates are expected to be successful. + EXPECT_TRUE(unpark_called); + + // Updates have been sent so this counter should remain 0. + EXPECT_EQ(0, service_->communication_state_->getUnsentUpdateCount()); + + // The server 2 should have received one command. + EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 2 has received lease6-bulk-apply command. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_TRUE(update_request2); + + // Lease updates should be successfully sent to server3. + EXPECT_EQ(1, factory3_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 3 has received lease6-bulk-apply command. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_TRUE(update_request3); + } + + /// @brief Tests that DHCPv6 lease updates are queued when the server is in the + /// communication-recovery state and later sent before transitioning back to + /// the load-balancing state. + void testSendUpdatesCommunicationRecovery6() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST)); + + // Packet shouldn't be unparked because no updates went out. We merely + // queued the updates. + EXPECT_FALSE(unpark_called); + + // Let's make sure they have been queued. + EXPECT_EQ(2, service_->lease_update_backlog_.size()); + + // Make partner available. + service_->communication_state_->poke(); + service_->communication_state_->setPartnerState("load-balancing"); + + // This should cause the server to send outstanding lease updates and + // because they are all successful the server should transition to the + // load-balancing state and continue normal operation. + testSynchronousCommands([this]() { + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); + }); + + // Bulk lease update should have been sent. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac")); + + // Backlog should be empty. + EXPECT_EQ(0, service_->lease_update_backlog_.size()); + } + + /// @brief Test the cases when the trying to recover from the communication + /// interruption and sending lease updates or/and ha-reset fails. + /// + /// @param partner_state partner state when communication is re-established. + /// @param lease_update_result control result returned to lease updates. + /// @param ha_reset_result control result returned to ha-reset command. + /// @param overflow boolean value indicating if this test should verify the + /// case when the leases backlog is overflown (when true), or not (when + /// false). + void testSendUpdatesCommunicationRecovery6Failed(const std::string& partner_state, + const int lease_update_result, + const int ha_reset_result, + const bool overflow = false) { + // Partner responds with a specified control result to lease updates. + factory2_->getResponseCreator()->setControlResult("lease6-bulk-apply", + lease_update_result); + // Partner returns specified control result to ha-reset. + factory2_->getResponseCreator()->setControlResult("ha-reset", + ha_reset_result); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST)); + + // Packet shouldn't be unparked because no updates went out. We merely + // queued the updates. + EXPECT_FALSE(unpark_called); + + // Let's make sure they have been queued. + EXPECT_EQ(2, service_->lease_update_backlog_.size()); + + // When testing the case when the backlog should be overflown, we need + // to add several more leases to the backlog to exceed the limit. + if (overflow) { + ASSERT_NO_THROW(generateTestLeases6()); + for (auto lease : leases6_) { + service_->lease_update_backlog_.push(LeaseUpdateBacklog::ADD, lease); + } + } + + // Make partner available. + service_->communication_state_->poke(); + service_->communication_state_->setPartnerState(partner_state); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This should cause the server to attempt to send outstanding lease + // updates to the partner. + testSynchronousCommands([this, ha_reset_result]() { + service_->runModel(HAService::NOP_EVT); + // If the ha-reset returns success the server should transition to the + // waiting state and begin synchronization. Otherwise, if the ha-reset + // fails the server should wait in the communication-recovery state + // until it succeeds. + if (ha_reset_result == CONTROL_RESULT_SUCCESS) { + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + } else { + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); + } + }); + + // The server will only send lease updates if it is not overflown. If + // it is overflown, it will rather transition to the waiting state to + // initiate full synchronization. + if (!overflow) { + // The server should have sent lease updates in a single command. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac")); + } + + if ((partner_state == "load-balancing") || (partner_state == "communication-recovery")) { + // The lease updates failed and the partner remains in the load-balancing or + // communication-recovery state, so the server should send ha-reset to the + // partner to cause it to transition to the waiting state and synchronize + // the lease database. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-reset", "")); + } else { + // The lease updates failed but the partner is already synchronizing lease + // database. In this case, don't send the ha-reset. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-reset", "")); + } + + // Backlog should be empty. + EXPECT_EQ(0, service_->lease_update_backlog_.size()); + } + + /// @brief Tests scenarios when lease updates are not sent to the failover peer. + void testSendUpdatesPartnerDown6() { + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 0, MyState(HA_PARTNER_DOWN_ST)); + + // There were no lease updates for which we have been waiting to + // complete so the packet was never unparked. Note that in such + // situation the packet is not parked either. + EXPECT_FALSE(unpark_called); + + // In the partner-down state we don't send lease updates. We + // should count transactions for which lease updates were not sent. + // This is later returned in the heartbeat so the partner can + // determine whether it should synchronize its lease database or + // not. + EXPECT_EQ(1, service_->communication_state_->getUnsentUpdateCount()); + + // Server 2 should not receive lease6-bulk-apply. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_FALSE(update_request2); + } + + /// @brief Tests scenarios when one of the servers to which + /// updates are sent is offline. + void testSendUpdatesActiveServerOffline6() { + // Start only two servers out of three. The server 3 is not running. + ASSERT_NO_THROW({ + listener_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates6([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // Server 2 should not receive lease6-bulk-apply. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_FALSE(update_request2); + } + + /// @brief Tests scenarios when one of the servers to which + /// updates are sent is offline. + void testSendUpdatesBackupServerOffline6() { + // Start only two servers out of three. The server 2 is not running. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 1); + + EXPECT_TRUE(unpark_called); + + // The server 2 should have received one command. + EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 2 has received lease6-bulk-apply command. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_TRUE(update_request2); + + // Server 3 should not receive lease6-bulk-apply. + auto update_request3 = + factory3_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_FALSE(update_request3); + } + + /// @brief Tests scenarios when one of the servers to which a + /// lease update is sent returns an error. + void testSendUpdatesControlResultError6() { + // Instruct the server 2 to return an error as a result of receiving a command. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates6([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // The updates should be sent to server 2 and this server should return error code. + EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Server 2 should receive lease6-bulk-apply. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_TRUE(update_request2); + } + + /// @brief Tests scenarios when one of the servers to which a + /// lease update is sent does not authorize the local server. + void testSendUpdatesControlResultUnauthorized6() { + // Instruct the server 2 to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates6([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, true, 1); + + // The updates should be sent to server 2 and this server should return error code. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); + } + + /// @brief These tests verify that the server accepts the response + /// to the lease6-bulk-apply command including + /// failed-deleted-leases and failed-leases parameters. + void testSendUpdatesFailedLeases6() { + // Create a dummy lease which failed to be deleted. + auto failed_deleted_lease = Element::createMap(); + failed_deleted_lease->set("type", + Element::create("IA_NA")); + failed_deleted_lease->set("ip-address", + Element::create("2001:db8:1::1")); + failed_deleted_lease->set("subnet-id", + Element::create(1)); + failed_deleted_lease->set("result", + Element::create(CONTROL_RESULT_EMPTY)); + failed_deleted_lease->set("error-message", + Element::create("no lease found")); + + // Crate a dummy lease which failed to be created. + auto failed_lease = Element::createMap(); + failed_lease->set("type", + Element::create("IA_PD")); + failed_lease->set("ip-address", + Element::create("2001:db8:1::")); + failed_lease->set("subnet-id", + Element::create(2)); + failed_lease->set("result", + Element::create(CONTROL_RESULT_ERROR)); + failed_lease->set("error-message", + Element::create("failed to create lease")); + + // Create the "failed-deleted-leases" list. + auto failed_deleted_leases = Element::createList(); + failed_deleted_leases->add(failed_deleted_lease); + + // Create the "failed-leases" list. + auto failed_leases = Element::createList(); + failed_leases->add(failed_lease); + + // Add both lists to the arguments. + ElementPtr arguments = Element::createMap(); + arguments->set("failed-deleted-leases", failed_deleted_leases); + arguments->set("failed-leases", failed_leases); + + // Configure the server to return this response. + factory2_->getResponseCreator()->setArguments("lease6-bulk-apply", + arguments); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // This flag will be set to true if unpark is called. + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { unpark_called = true; }, + false, 1); + + // Expecting that the packet was unparked because lease + // updates are expected to be successful. + EXPECT_TRUE(unpark_called); + + // The server 2 should have received one command. + EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size()); + + // Check that the server 2 has received lease6-bulk-apply command. + auto update_request2 = + factory2_->getResponseCreator()->findRequest("lease6-bulk-apply", + "2001:db8:1::cafe", + "2001:db8:1::efac"); + EXPECT_TRUE(update_request2); + } + + /// @brief Test the scenario when the servers receiving a lease update + /// return the conflict status code. + void testSendUpdatesControlResultConflict6() { + // Crate a dummy lease for which a conflict status code is returned. + auto failed_lease = Element::createMap(); + failed_lease->set("type", + Element::create("IA_PD")); + failed_lease->set("ip-address", + Element::create("2001:db8:1::")); + failed_lease->set("subnet-id", + Element::create(2)); + failed_lease->set("result", + Element::create(CONTROL_RESULT_CONFLICT)); + failed_lease->set("error-message", + Element::create("failed to create lease")); + + // Create the "failed-leases" list. + auto failed_leases = Element::createList(); + failed_leases->add(failed_lease); + + // Add the list to the arguments. + ElementPtr arguments = Element::createMap(); + arguments->set("failed-leases", failed_leases); + + // Active server returns an empty status code to indicate that no lease + // has been created. It should cause the HA service to look into the + // status codes for individual leases. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_EMPTY); + factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT); + + // Configure the server to return our response. + factory2_->getResponseCreator()->setArguments("lease6-bulk-apply", + arguments); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + testSendLeaseUpdates6([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, false, 1); + + // Ensure that the server has recorded the lease update conflict. The conflict + // reported by the backup server should not count. + EXPECT_EQ(1, service_->communication_state_->getRejectedLeaseUpdatesCount()); + + // Change the active server's response to success. The initially rejected + // lease update should no longer be tracked. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_SUCCESS); + + bool unpark_called = false; + testSendLeaseUpdates6([&unpark_called] { + unpark_called = true; + }, false, 1, MyState(HA_LOAD_BALANCING_ST), true, false); + EXPECT_TRUE(unpark_called); + EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount()); + } + + /// @brief Test the scenario when the server receiving a lease update returns + /// both the conflict and error status code. The latter should take precedence. + void testSendUpdatesControlResultConflict6ErrorPrecedence() { + // Create the "failed-leases" list. + auto failed_leases = Element::createList(); + + // Crate a dummy lease for which a conflict status code is returned. + auto failed_lease = Element::createMap(); + failed_lease->set("type", + Element::create("IA_NA")); + failed_lease->set("ip-address", + Element::create("2001:db8:1::")); + failed_lease->set("subnet-id", + Element::create(2)); + failed_lease->set("result", + Element::create(CONTROL_RESULT_CONFLICT)); + failed_lease->set("error-message", + Element::create("failed to create lease")); + failed_leases->add(failed_lease); + + // Create another lease with an error status code. + failed_lease = Element::createMap(); + failed_lease->set("type", + Element::create("IA_NA")); + failed_lease->set("ip-address", + Element::create("2001:db8:1::2")); + failed_lease->set("subnet-id", + Element::create(2)); + failed_lease->set("result", + Element::create(CONTROL_RESULT_ERROR)); + failed_lease->set("error-message", + Element::create("failed to create lease")); + failed_leases->add(failed_lease); + + // Add the list to the arguments. + ElementPtr arguments = Element::createMap(); + arguments->set("failed-leases", failed_leases); + + // Active server returns an empty status code to indicate that no lease + // has been created. It should cause the HA service to look into the + // status codes for individual leases. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_EMPTY); + + // Configure the server to return our response. + factory2_->getResponseCreator()->setArguments("lease6-bulk-apply", + arguments); + + // Start HTTP servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + testSendLeaseUpdates6([] { + ADD_FAILURE() << "unpark function called but expected that " + "the packet is dropped"; + }, false, 1); + + // The conflict should not be recorded because the error status code + // takes precedence. + EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount()); + } + + /// @brief Runs HAService::processSynchronize for the DHCPv4 server and + /// returns a response. + /// + /// The HAService::processSynchronize is synchronous. Therefore, the IO service + /// for HTTP servers is run in a thread. The unit test is responsible for setting + /// up the status codes to be returned by the servers, verifying a response and + /// leases in the lease database. + /// + /// @param [out] rsp pointer to the object where response will be stored. + void runProcessSynchronize4(ConstElementPtr& rsp) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases4()); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Leases are fetched in pages, so the lease4-get-page should be + // sent multiple times. The server is configured to return leases + // in 3-element chunks. + createPagedSyncResponses4(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in bakckground to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-sync command. + ASSERT_NO_THROW(rsp = service.processSynchronize("server2", 20)); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + } + + /// @brief Runs HAService::processSynchronize for the DHCPv6 server + /// and returns a response. + /// + /// The HAService::processSynchronize is synchronous. Therefore, the IO service + /// for HTTP servers is run in a thread. The unit test is responsible for setting + /// up the status codes to be returned by the servers, verifying a response and + /// leases in the lease database. + /// + /// @param [out] rsp pointer to the object where response will be stored. + void runProcessSynchronize6(ConstElementPtr& rsp) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases6()); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Leases are fetched in pages, so the lease6-get-page should be + // sent multiple times. The server is configured to return leases + // in 3-element chunks. + createPagedSyncResponses6(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + HAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + + // The tested function is synchronous, so we need to run server side IO service + // in bakckground to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-sync command. + ASSERT_NO_THROW(rsp = service.processSynchronize("server2", 20)); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + } + + /// @brief Pointer to the HA service under test. + TestHAServicePtr service_; + + /// @brief HTTP response factory for server 1. + TestHttpResponseCreatorFactoryPtr factory_; + + /// @brief HTTP response factory for server 2. + TestHttpResponseCreatorFactoryPtr factory2_; + + /// @brief HTTP response factory for server 3. + TestHttpResponseCreatorFactoryPtr factory3_; + + /// @brief Test HTTP server 1. + HttpListenerPtr listener_; + + /// @brief Test HTTP server 2. + HttpListenerPtr listener2_; + + /// @brief Test HTTP server 3. + HttpListenerPtr listener3_; + + /// @brief IPv4 leases to be used in the tests. + std::vector leases4_; + + /// @brief IPv6 leases to be used in the tests. + std::vector leases6_; + + /// @brief Basic HTTP authentication user id for server1. + std::string user1_; + + /// @brief Basic HTTP authentication password for server1. + std::string password1_; + + /// @brief Basic HTTP authentication user id for server2. + std::string user2_; + + /// @brief Basic HTTP authentication password for server2. + std::string password2_; + + /// @brief Basic HTTP authentication user id for server3. + std::string user3_; + + /// @brief Basic HTTP authentication password for server3. + std::string password3_; +}; + +// Test that server performs load balancing and assigns appropriate classes +// to the queries. +TEST_F(HAServiceTest, loadBalancingScopeSelection) { + // Create HA configuration for load balancing. + HAConfigPtr config_storage = createValidConfiguration(); + // ... and HA service using this configuration. + TestHAService service(io_service_, network_state_, config_storage); + service.verboseTransition(HA_LOAD_BALANCING_ST); + service.runModel(HAService::NOP_EVT); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + + // Some queries should be processed by this server, some not. + if (service.inScope(query4)) { + // If the query is to be processed by this server the query + // should be assigned to the "HA_server1" class but not to + // the "HA_server2" class. + ASSERT_TRUE(query4->inClass(ClientClass("HA_server1"))); + ASSERT_FALSE(query4->inClass(ClientClass("HA_server2"))); + ++in_scope; + + } else { + // If the query is to be processed by another server, the + // "HA_server2" class should be assigned instead. + ASSERT_FALSE(query4->inClass(ClientClass("HA_server1"))); + ASSERT_TRUE(query4->inClass(ClientClass("HA_server2"))); + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); +} + +// Test that primary server in hot standby configuration processes all queries. +TEST_F(HAServiceTest, hotStandbyScopeSelectionThisPrimary) { + HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY); + config_storage->getPeerConfig("server2")->setRole("standby"); + + // ... and HA service using this configuration. + TestHAService service(io_service_, network_state_, config_storage); + service.verboseTransition(HA_HOT_STANDBY_ST); + service.runModel(HAService::NOP_EVT); + + // Check the reported info about servers. + ConstElementPtr ha_servers = service.processStatusGet(); + ASSERT_TRUE(ha_servers); + std::string expected = "{" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ \"server1\" ]," + " \"state\": \"hot-standby\"" + " }, " + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"role\": \"standby\"," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"," + " \"communication-interrupted\": false," + " \"connecting-clients\": 0," + " \"unacked-clients\": 0," + " \"unacked-clients-left\": 0," + " \"analyzed-packets\": 0" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + + // All queries should be processed by this server. + ASSERT_TRUE(service.inScope(query4)); + // The query should be assigned to the "HA_server1" class but not to + // the "HA_server2" class. + ASSERT_TRUE(query4->inClass(ClientClass("HA_server1"))); + ASSERT_FALSE(query4->inClass(ClientClass("HA_server2"))); + } +} + +// Test that secondary server in hot standby configuration processes no queries. +TEST_F(HAServiceTest, hotStandbyScopeSelectionThisStandby) { + HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY); + config_storage->getPeerConfig("server2")->setRole("standby"); + config_storage->setThisServerName("server2"); + + // ... and HA service using this configuration. + TestHAService service(io_service_, network_state_, config_storage); + + // Check the reported info about servers. + ConstElementPtr ha_servers = service.processStatusGet(); + ASSERT_TRUE(ha_servers); + + std::string expected = "{" + " \"local\": {" + " \"role\": \"standby\"," + " \"scopes\": [ ]," + " \"state\": \"waiting\"" + " }, " + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"role\": \"primary\"," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"," + " \"communication-interrupted\": false," + " \"connecting-clients\": 0," + " \"unacked-clients\": 0," + " \"unacked-clients-left\": 0," + " \"analyzed-packets\": 0" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + + // No queries should be processed by this server. + ASSERT_FALSE(service.inScope(query4)); + // The query should be assigned to the "HA_server1" class but not to + // the "HA_server2" class. + ASSERT_TRUE(query4->inClass(ClientClass("HA_server1"))); + ASSERT_FALSE(query4->inClass(ClientClass("HA_server2"))); + } +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdates) { + testSendSuccessfulUpdates(); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdatesMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendSuccessfulUpdates(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state for later send. +TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery) { + testSendUpdatesCommunicationRecovery(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state for later send. Multi threading case. +TEST_F(HAServiceTest, sendUpdatesCommunicationRecoveryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecovery(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in load-balancing +// state when the communication is re-established, so the test expects that the +// ha-reset command is sent to the partner. +TEST_F(HAServiceTest, communicationRecoveryFailedPartnerLoadBalancing) { + testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in load-balancing +// state when the communication is re-established, so the test expects that the +// ha-reset command is sent to the partner. +TEST_F(HAServiceTest, communicationRecoveryFailedPartnerLoadBalancingMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later fails. The partner server in the load-balancing +// state but sending ha-reset fails. The server should remain in the +// communication-recovery state. +TEST_F(HAServiceTest, communicationRecoveryFailedResetFailed) { + testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_ERROR); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later fails. The partner server in the load-balancing +// state but sending ha-reset fails. The server should remain in the +// communication-recovery state. +TEST_F(HAServiceTest, communicationRecoveryFailedResetFailedMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_ERROR); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in ready state +// when the communication is re-established, so the test expects that the +// ha-reset command is NOT sent to the partner. The lease backlog is overflown, +// so the server should transition to the waiting state. +TEST_F(HAServiceTest, communicationRecoveryFailedPartnerReady) { + testSendUpdatesCommunicationRecoveryFailed("ready", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS, true); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in ready state +// when the communication is re-established, so the test expects that the +// ha-reset command is NOT sent to the partner. The lease backlog is overflown, +// so the server should transition to the waiting state. +TEST_F(HAServiceTest, communicationRecoveryFailedPartnerReadyMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecoveryFailed("ready", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS, true); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdatesAuthorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + testSendSuccessfulUpdates(); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdatesAuthorizedMultiThreading) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + MultiThreadingMgr::instance().setMode(true); + testSendSuccessfulUpdates(); +} + +// Test scenario when lease updates are not sent to the failover peer. +TEST_F(HAServiceTest, sendUpdatesPartnerDown) { + testSendUpdatesPartnerDown(); +} + +// Test scenario when lease updates are not sent to the failover peer. +TEST_F(HAServiceTest, sendUpdatesPartnerDownMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesPartnerDown(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesActiveServerOffline) { + testSendUpdatesActiveServerOffline(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesActiveServerOfflineMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesActiveServerOffline(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesBackupServerOffline) { + testSendUpdatesBackupServerOffline(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesBackupServerOfflineMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesBackupServerOffline(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns an error. +TEST_F(HAServiceTest, sendUpdatesControlResultError) { + testSendUpdatesControlResultError(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns an error. +TEST_F(HAServiceTest, sendUpdatesControlResultErrorMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultError(); +} + +// Test scenario when one of the servers to which a lease update is sent +// requires not provided authentication. +TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized) { + testSendUpdatesControlResultUnauthorized(); +} + +// Test scenario when one of the servers to which a lease update is sent +// requires not provided authentication. +TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorizedMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultUnauthorized(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns a conflict status code. +TEST_F(HAServiceTest, sendUpdatesControlResultConflict) { + testSendUpdatesControlResultConflict(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns a conflict status code. +TEST_F(HAServiceTest, sendUpdatesControlResultConflictMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultConflict(); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdates6) { + testSendSuccessfulUpdates6(); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdates6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendSuccessfulUpdates6(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state for later send. Then, the partner refuses lease updates causing the +// server to transition to the waiting state. +TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery6) { + testSendUpdatesCommunicationRecovery6(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state for later send. Then, the partner refuses lease updates causing the +// server to transition to the waiting state. +TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecovery6(); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in load-balancing +// state when the communication is re-established, so the test expects that the +// ha-reset command is sent to the partner. +TEST_F(HAServiceTest, communicationRecoveryFailed6PartnerLoadBalancing) { + testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in load-balancing +// state when the communication is re-established, so the test expects that the +// ha-reset command is sent to the partner. +TEST_F(HAServiceTest, communicationRecovery6FailedPartnerLoadBalancingMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later fails. The partner server in the load-balancing +// state but sending ha-reset fails. The server should remain in the +// communication-recovery state. +TEST_F(HAServiceTest, communicationRecovery6FailedResetFailed) { + testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_ERROR); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later fails. The partner server in the load-balancing +// state but sending ha-reset fails. The server should remain in the +// communication-recovery state. +TEST_F(HAServiceTest, communicationRecovery6FailedResetFailedMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR, + CONTROL_RESULT_ERROR); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in ready state +// when the communication is re-established, so the test expects that the +// ha-reset command is NOT sent to the partner. The lease backlog is overflown, +// so the server should transition to the waiting state. +TEST_F(HAServiceTest, communicationRecovery6FailedPartnerReady) { + testSendUpdatesCommunicationRecovery6Failed("ready", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS, true); +} + +// Test scenario when lease updates are queued in the communication-recovery +// state and sending them later is unsuccessful. Partner is in ready state +// when the communication is re-established, so the test expects that the +// ha-reset command is NOT sent to the partner. The lease backlog is overflown, +// so the server should transition to the waiting state. +TEST_F(HAServiceTest, communicationRecovery6FailedPartnerReadyMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesCommunicationRecovery6Failed("ready", CONTROL_RESULT_ERROR, + CONTROL_RESULT_SUCCESS, true); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdates6Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + testSendSuccessfulUpdates6(); +} + +// Test scenario when all lease updates are sent successfully. +TEST_F(HAServiceTest, sendSuccessfulUpdates6AuthorizedMultiThreading) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + MultiThreadingMgr::instance().setMode(true); + testSendSuccessfulUpdates6(); +} + +// Test scenario when lease updates are not sent to the failover peer. +TEST_F(HAServiceTest, sendUpdatesPartnerDown6) { + testSendUpdatesPartnerDown6(); +} + +// Test scenario when lease updates are not sent to the failover peer. +TEST_F(HAServiceTest, sendUpdatesPartnerDown6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesPartnerDown6(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesActiveServerOffline6) { + testSendUpdatesActiveServerOffline6(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesActiveServerOffline6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesActiveServerOffline6(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesBackupServerOffline6) { + testSendUpdatesBackupServerOffline6(); +} + +// Test scenario when one of the servers to which updates are sent is offline. +TEST_F(HAServiceTest, sendUpdatesBackupServerOffline6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesBackupServerOffline6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns an error. +TEST_F(HAServiceTest, sendUpdatesControlResultError6) { + testSendUpdatesControlResultError6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns an error. +TEST_F(HAServiceTest, sendUpdatesControlResultError6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultError6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// requires not provided authentication. +TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized6) { + testSendUpdatesControlResultUnauthorized6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// requires not provided authentication. +TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultUnauthorized6(); +} + +// This test verifies that the server accepts the response to the lease6-bulk-apply +// command including failed-deleted-leases and failed-leases parameters. +TEST_F(HAServiceTest, sendUpdatesFailedLeases6) { + testSendUpdatesFailedLeases6(); +} + +// This test verifies that the server accepts the response to the lease6-bulk-apply +// command including failed-deleted-leases and failed-leases parameters. +TEST_F(HAServiceTest, sendUpdatesFailedLeases6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesFailedLeases6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns a conflict status code. +TEST_F(HAServiceTest, sendUpdatesControlResultConflict6) { + testSendUpdatesControlResultConflict6(); +} + +// Test scenario when one of the servers to which a lease update is sent +// returns a conflict status code. +TEST_F(HAServiceTest, sendUpdatesControlResultConflict6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultConflict6(); +} + +// Test the scenario when the server receiving a lease update returns +// both the conflict and error status code. The latter should take precedence. +TEST_F(HAServiceTest, sendUpdatesControlResultConflict6ErrorPrecedence) { + testSendUpdatesControlResultConflict6ErrorPrecedence(); +} + + // Test the scenario when the server receiving a lease update returns +// both the conflict and error status code. The latter should take precedence. +TEST_F(HAServiceTest, sendUpdatesControlResultConflict6ErrorPrecedenceMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + testSendUpdatesControlResultConflict6ErrorPrecedence(); +} + +// This test verifies that the heartbeat command is processed successfully. +TEST_F(HAServiceTest, processHeartbeat) { + // Create HA configuration for 3 servers. This server is + // server 1. + std::string config_text = + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"load-balancing\"," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:18123/\"," + " \"role\": \"primary\"," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:18124/\"," + " \"role\": \"secondary\"," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:18125/\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }" + " ]" + " }" + "]"; + + // Parse the HA configuration. + HAConfigPtr config_storage(new HAConfig()); + HAConfigParser parser; + ASSERT_NO_THROW(parser.parse(config_storage, Element::fromJSON(config_text))); + + TestHAService service(io_service_, network_state_, config_storage); + service.query_filter_.serveDefaultScopes(); + + int unsent_updates = 6; + for (auto i = 0; i < unsent_updates; ++i) { + service.communication_state_->increaseUnsentUpdateCount(); + } + + // Process heartbeat command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processHeartbeat()); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA peer status returned."); + // Response must include arguments. + ConstElementPtr args = rsp->get("arguments"); + ASSERT_TRUE(args); + + // Response must include server state as string. + ConstElementPtr state = args->get("state"); + ASSERT_TRUE(state); + EXPECT_EQ(Element::string, state->getType()); + EXPECT_EQ("waiting", state->stringValue()); + + // Response must include timestamp when the response was generated. + ConstElementPtr date_time = args->get("date-time"); + ASSERT_TRUE(date_time); + EXPECT_EQ(Element::string, date_time->getType()); + + auto scopes_list = args->get("scopes"); + ASSERT_TRUE(scopes_list); + EXPECT_EQ(Element::list, scopes_list->getType()); + ASSERT_EQ(1, scopes_list->size()); + auto scope = scopes_list->get(0); + ASSERT_TRUE(scope); + EXPECT_EQ(Element::string, scope->getType()); + EXPECT_EQ("server1", scope->stringValue()); + + // The response should contain the timestamp in the format specified + // in RFC1123. We use the HttpDateTime method to parse this timestamp. + HttpDateTime t; + ASSERT_NO_THROW(t = HttpDateTime::fromRfc1123(date_time->stringValue())); + + // Let's test if the timestamp is accurate. We do it by checking current + // time and comparing with the received timestamp. + HttpDateTime now; + boost::posix_time::time_duration td = now.getPtime() - t.getPtime(); + + // Let's allow the response propagation time of 5 seconds to make + // sure this test doesn't fail on slow systems. + EXPECT_LT(td.seconds(), 5); + + // The response should contain unsent-update-count parameter indicating + // how many updates haven't been sent to a partner because the partner + // was unavailable. + ConstElementPtr unsent_update_count = args->get("unsent-update-count"); + ASSERT_TRUE(unsent_update_count); + EXPECT_EQ(Element::integer, unsent_update_count->getType()); + EXPECT_EQ(unsent_updates, static_cast(unsent_update_count->intValue())); +} + +// This test verifies that the correct value of the heartbeat-delay is used. +TEST_F(HAServiceTest, recurringHeartbeatDelay) { + HAConfigPtr config_storage = createValidConfiguration(); + + // Set the heartbeat delay to 6 seconds. + config_storage->setHeartbeatDelay(6000); + + // The communication state is the member of the HAServce object. We have to + // replace this object with our own implementation to have an ability to + // test the setup of the interval timer. + NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_, + config_storage)); + + TestHAService service(io_service_, network_state_, config_storage); + service.communication_state_ = state; + + // Let's explicitly transition the state machine to the load balancing state + // in which the periodic heartbeats should be generated. + ASSERT_NO_THROW(service.verboseTransition(HA_LOAD_BALANCING_ST)); + ASSERT_NO_THROW(service.runModel(HAService::NOP_EVT)); + + ASSERT_TRUE(state->timer_); + EXPECT_EQ(6000, state->timer_->getInterval()); +} + +// This test verifies that the heartbeat is periodically sent to the +// other server. +TEST_F(HAServiceTest, recurringHeartbeat) { + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // All servers are configured to return success and all servers are online. + // The heartbeat should be successful (as indicated by the 'true' argument). + ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, true)); + + // Server 2 should have received the heartbeat + EXPECT_GE(factory2_->getResponseCreator()->getReceivedRequests().size(), 0); +} + +// This test verifies that the heartbeat is periodically sent to the +// other server. +TEST_F(HAServiceTest, recurringHeartbeatAuthorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // All servers are configured to return success and all servers are online. + // The heartbeat should be successful (as indicated by the 'true' argument). + ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, true)); + + // Server 2 should have received the heartbeat + EXPECT_GE(factory2_->getResponseCreator()->getReceivedRequests().size(), 0); +} + +// This test verifies that the heartbeat is considered being unsuccessful if the +// partner is offline. +TEST_F(HAServiceTest, recurringHeartbeatServerOffline) { + // Start the servers but do not start server 2. + ASSERT_NO_THROW({ + listener_->start(); + listener3_->start(); + }); + + // The servers are configured to return success but the server 2 is offline + // so the heartbeat should be unsuccessful. + ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, false)); + + // Server 2 is offline so it would be very weird if it received any command. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); +} + +// This test verifies that the heartbeat is considered being unsuccessful if the +// partner returns error control result. +TEST_F(HAServiceTest, recurringHeartbeatControlResultError) { + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // Run the actual test. The servers return a control error and it is expected + // that the state is not poked. + ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_ERROR, false)); + + // Server 2 should have received the heartbeat. + EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size()); +} + +// This test verifies that the heartbeat is considered being unsuccessful if +// the partner requires not provided authentication. +TEST_F(HAServiceTest, recurringHeartbeatControlResultUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + // Run the actual test. The servers return a control error and it is expected + // that the state is not poked. + ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_ERROR, false)); + + // Server 2 should have received the heartbeat. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); +} + +// This test verifies that IPv4 leases can be fetched from the peer and inserted +// or updated in the local lease database. +TEST_F(HAServiceTest, asyncSyncLeases4) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases4()); + + for (size_t i = 0; i < leases4_.size(); ++i) { + // For every even lease index we add this lease to the database to exercise + // the scenario when a lease is already in the database and may be updated + // by the lease synchronization procedure. + if ((i % 2) == 0) { + // Add a copy of the lease to make sure that by modifying the lease + // contents we don't affect the lease in the database. + Lease4Ptr lease_to_add(new Lease4(*leases4_[i])); + // Modify valid lifetime of the lease in the database so we can + // later use this value to verify if the lease has been updated. + --lease_to_add->valid_lft_; + LeaseMgrFactory::instance().addLease(lease_to_add); + } + } + + // Modify cltt of the first lease. This lease should be updated as a result + // of synchronization process because cltt is checked and the lease is + // updated if the cltt of the fetched lease is later than the cltt of the + // existing lease. + ++leases4_[0]->cltt_; + + // For the second lease, set the wrong subnet identifier. This should be + // rejected and this lease shouldn't be inserted into the database. + // Other leases should be inserted/updated just fine. + ++leases4_[1]->subnet_id_ = 0; + + // Modify the partner's lease cltt so it is earlier than the local lease. + // Therefore, this lease update should be rejected. + --leases4_[2]->cltt_; + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Convert leases to the JSON format, the same as used by the lease_cmds + // hook library. Configure our test HTTP servers to return those + // leases in this format. + ElementPtr response_arguments = Element::createMap(); + + // Leases are fetched in pages, so the lease4-get-page should be + // sent multiple times. The server is configured to return leases + // in 3-element chunks. + createPagedSyncResponses4(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() { + // Stop running the IO service if we see a lease in the lease + // database which is expected to be inserted as a result of lease + // syncing. + return (!LeaseMgrFactory::instance().getLeases4(SubnetID(10)).empty()); + })); + + // Check if all leases have been stored in the local database. + for (size_t i = 0; i < leases4_.size(); ++i) { + if (i == 1) { + // This lease was purposely malformed and thus shouldn't be + // inserted into the database. + EXPECT_FALSE(LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_)) + << "lease " << leases4_[i]->addr_.toText() + << " was inserted into the database, but it shouldn't"; + + } else { + // All other leases should be in the database. + Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText() + << " not in the lease database"; + // The lease with #2 returned by the partner is older than its local instance. + // The local server should reject this lease. + if (i == 2) { + // The existing lease should have unmodified timestamp because the + // update is expected to be rejected. Same for valid lifetime. + EXPECT_LT(leases4_[i]->cltt_, existing_lease->cltt_); + EXPECT_NE(leases4_[i]->valid_lft_, existing_lease->valid_lft_); + + } else { + // All other leases should have the same cltt. + EXPECT_EQ(leases4_[i]->cltt_, existing_lease->cltt_); + + // Leases with even indexes were added to the database with modified + // valid lifetime. Thus the local copy of each such lease should have + // this modified valid lifetime. The lease #0 should be updated from + // the partner because of the partner's cltt was set to later time. + if ((i != 0) && (i % 2) == 0) { + EXPECT_EQ(leases4_[i]->valid_lft_ - 1, existing_lease->valid_lft_); + + } else { + // All other leases should have been fetched from the partner and + // inserted with no change. + EXPECT_EQ(leases4_[i]->valid_lft_, existing_lease->valid_lft_); + } + } + } + } +} + +// This test verifies that IPv4 leases can be fetched from the peer and inserted +// or updated in the local lease database. +TEST_F(HAServiceTest, asyncSyncLeases4Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases4()); + + for (size_t i = 0; i < leases4_.size(); ++i) { + // For every even lease index we add this lease to the database to exercise + // the scenario when a lease is already in the database and may be updated + // by the lease synchronization procedure. + if ((i % 2) == 0) { + // Add a copy of the lease to make sure that by modifying the lease + // contents we don't affect the lease in the database. + Lease4Ptr lease_to_add(new Lease4(*leases4_[i])); + // Modify valid lifetime of the lease in the database so we can + // later use this value to verify if the lease has been updated. + --lease_to_add->valid_lft_; + LeaseMgrFactory::instance().addLease(lease_to_add); + } + } + + // Modify cltt of the first lease. This lease should be updated as a result + // of synchronization process because cltt is checked and the lease is + // updated if the cltt of the fetched lease is later than the cltt of the + // existing lease. + ++leases4_[0]->cltt_; + + // For the second lease, set the wrong subnet identifier. This should be + // rejected and this lease shouldn't be inserted into the database. + // Other leases should be inserted/updated just fine. + ++leases4_[1]->subnet_id_ = 0; + + // Modify the partner's lease cltt so it is earlier than the local lease. + // Therefore, this lease update should be rejected. + --leases4_[2]->cltt_; + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Convert leases to the JSON format, the same as used by the lease_cmds + // hook library. Configure our test HTTP servers to return those + // leases in this format. + ElementPtr response_arguments = Element::createMap(); + + // Leases are fetched in pages, so the lease4-get-page should be + // sent multiple times. The server is configured to return leases + // in 3-element chunks. + createPagedSyncResponses4(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() { + // Stop running the IO service if we see a lease in the lease + // database which is expected to be inserted as a result of lease + // syncing. + return (!LeaseMgrFactory::instance().getLeases4(SubnetID(10)).empty()); + })); + + // Check if all leases have been stored in the local database. + for (size_t i = 0; i < leases4_.size(); ++i) { + if (i == 1) { + // This lease was purposely malformed and thus shouldn't be + // inserted into the database. + EXPECT_FALSE(LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_)) + << "lease " << leases4_[i]->addr_.toText() + << " was inserted into the database, but it shouldn't"; + + } else { + // All other leases should be in the database. + Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText() + << " not in the lease database"; + // The lease with #2 returned by the partner is older than its local instance. + // The local server should reject this lease. + if (i == 2) { + // The existing lease should have unmodified timestamp because the + // update is expected to be rejected. Same for valid lifetime. + EXPECT_LT(leases4_[i]->cltt_, existing_lease->cltt_); + EXPECT_NE(leases4_[i]->valid_lft_, existing_lease->valid_lft_); + + } else { + // All other leases should have the same cltt. + EXPECT_EQ(leases4_[i]->cltt_, existing_lease->cltt_); + + // Leases with even indexes were added to the database with modified + // valid lifetime. Thus the local copy of each such lease should have + // this modified valid lifetime. The lease #0 should be updated from + // the partner because of the partner's cltt was set to later time. + if ((i != 0) && (i % 2) == 0) { + EXPECT_EQ(leases4_[i]->valid_lft_ - 1, existing_lease->valid_lft_); + + } else { + // All other leases should have been fetched from the partner and + // inserted with no change. + EXPECT_EQ(leases4_[i]->valid_lft_, existing_lease->valid_lft_); + } + } + } + } +} + +// Test that there is no exception thrown during leases synchronization +// when server returns a wrong answer. +TEST_F(HAServiceTest, asyncSyncLeases4WrongAnswer) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases4()); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Set empty response. This should cause the HA service to log an + // error but not crash. + ElementPtr response_arguments = Element::createMap(); + + factory2_->getResponseCreator()->setArguments(response_arguments); + factory3_->getResponseCreator()->setArguments(response_arguments); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(1000)); +} + +// Test that there is no exception thrown during leases synchronization +// when servers are offline. +TEST_F(HAServiceTest, asyncSyncLeases4ServerOffline) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + TestHAService service(io_service_, network_state_, config_storage); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service for 1 second. + ASSERT_NO_THROW(runIOService(1000)); +} + +// Test that there is no exception thrown during leases synchronization +// when servers require not provided authentication. +TEST_F(HAServiceTest, asyncSyncLeases4ServerUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false")); + + // Create IPv4 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases4()); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service for 1 second. + ASSERT_NO_THROW(runIOService(1000)); +} + +// This test verifies that IPv6 leases can be fetched from the peer and inserted +// or updated in the local lease database. +TEST_F(HAServiceTest, asyncSyncLeases6) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false")); + + // Create IPv6 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases6()); + + for (size_t i = 0; i < leases6_.size(); ++i) { + // For every even lease index we add this lease to the database to exercise + // the scenario when a lease is already in the database and may be updated + // by the lease synchronization procedure. + if ((i % 2) == 0) { + // Add a copy of the lease to make sure that by modifying the lease + // contents we don't affect the lease in the database. + Lease6Ptr lease_to_add(new Lease6(*leases6_[i])); + // Modify valid lifetime of the lease in the database so we can + // later use this value to verify if the lease has been updated. + --lease_to_add->valid_lft_; + LeaseMgrFactory::instance().addLease(lease_to_add); + } + } + + // Modify cltt of the first lease. This lease should be updated as a result + // of synchronization process because cltt is checked and the lease is + // updated if the cltt of the fetched lease is later than the cltt of the + // existing lease. + ++leases6_[0]->cltt_; + + // For the second lease, set the wrong subnet identifier. This should be + // rejected and this lease shouldn't be inserted into the database. + // Other leases should be inserted/updated just fine. + ++leases6_[1]->subnet_id_ = 0; + + // Modify the partner's lease cltt so it is earlier than the local lease. + // Therefore, this lease update should be rejected. + --leases6_[2]->cltt_; + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Convert leases to the JSON format, the same as used by the lease_cmds + // hook library. Configure our test HTTP servers to return those + // leases in this format. + ElementPtr response_arguments = Element::createMap(); + + // Leases are fetched in pages, so the lease4-get-page should be + // sent multiple times. We need to configure the server side to + // return leases in 3-element chunks. + createPagedSyncResponses6(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() { + // Stop running the IO service if we see a lease in the lease + // database which is expected to be inserted as a result of lease + // syncing. + return (!LeaseMgrFactory::instance().getLeases6(SubnetID(10)).empty()); + })); + + // Check if all leases have been stored in the local database. + for (size_t i = 0; i < leases6_.size(); ++i) { + if (i == 1) { + // This lease was purposely malformed and thus shouldn't be + // inserted into the database. + EXPECT_FALSE(LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_)) + << "lease " << leases6_[i]->addr_.toText() + << " was inserted into the database, but it shouldn't"; + } else { + // Other leases should be inserted/updated. + Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText() + << " not in the lease database"; + + if (i == 2) { + // The existing lease should have unmodified timestamp because the + // update is expected to be rejected. Same for valid lifetime. + EXPECT_LT(leases6_[i]->cltt_, existing_lease->cltt_); + EXPECT_NE(leases6_[i]->valid_lft_, existing_lease->valid_lft_); + + } else { + // All other leases should have the same cltt. + EXPECT_EQ(leases6_[i]->cltt_, existing_lease->cltt_); + + // Leases with even indexes were added to the database with modified + // valid lifetime. Thus the local copy of each such lease should have + // this modified valid lifetime. The lease #0 should be updated from + // the partner because of the partner's cltt was set to later time. + if ((i != 0) && (i % 2) == 0) { + EXPECT_EQ(leases6_[i]->valid_lft_ - 1, existing_lease->valid_lft_); + + } else { + // All other leases should have been fetched from the partner and + // inserted with no change. + EXPECT_EQ(leases6_[i]->valid_lft_, existing_lease->valid_lft_); + } + } + } + } +} + +// This test verifies that IPv6 leases can be fetched from the peer and inserted +// or updated in the local lease database. +TEST_F(HAServiceTest, asyncSyncLeases6Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false")); + + // Create IPv6 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases6()); + + for (size_t i = 0; i < leases6_.size(); ++i) { + // For every even lease index we add this lease to the database to exercise + // the scenario when a lease is already in the database and may be updated + // by the lease synchronization procedure. + if ((i % 2) == 0) { + // Add a copy of the lease to make sure that by modifying the lease + // contents we don't affect the lease in the database. + Lease6Ptr lease_to_add(new Lease6(*leases6_[i])); + // Modify valid lifetime of the lease in the database so we can + // later use this value to verify if the lease has been updated. + --lease_to_add->valid_lft_; + LeaseMgrFactory::instance().addLease(lease_to_add); + } + } + + // Modify cltt of the first lease. This lease should be updated as a result + // of synchronization process because cltt is checked and the lease is + // updated if the cltt of the fetched lease is later than the cltt of the + // existing lease. + ++leases6_[0]->cltt_; + + // For the second lease, set the wrong subnet identifier. This should be + // rejected and this lease shouldn't be inserted into the database. + // Other leases should be inserted/updated just fine. + ++leases6_[1]->subnet_id_ = 0; + + // Modify the partner's lease cltt so it is earlier than the local lease. + // Therefore, this lease update should be rejected. + --leases6_[2]->cltt_; + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Convert leases to the JSON format, the same as used by the lease_cmds + // hook library. Configure our test HTTP servers to return those + // leases in this format. + ElementPtr response_arguments = Element::createMap(); + + // Leases are fetched in pages, so the lease4-get-page should be + // sent multiple times. We need to configure the server side to + // return leases in 3-element chunks. + createPagedSyncResponses6(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() { + // Stop running the IO service if we see a lease in the lease + // database which is expected to be inserted as a result of lease + // syncing. + return (!LeaseMgrFactory::instance().getLeases6(SubnetID(10)).empty()); + })); + + // Check if all leases have been stored in the local database. + for (size_t i = 0; i < leases6_.size(); ++i) { + if (i == 1) { + // This lease was purposely malformed and thus shouldn't be + // inserted into the database. + EXPECT_FALSE(LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_)) + << "lease " << leases6_[i]->addr_.toText() + << " was inserted into the database, but it shouldn't"; + } else { + // Other leases should be inserted/updated. + Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText() + << " not in the lease database"; + + if (i == 2) { + // The existing lease should have unmodified timestamp because the + // update is expected to be rejected. Same for valid lifetime. + EXPECT_LT(leases6_[i]->cltt_, existing_lease->cltt_); + EXPECT_NE(leases6_[i]->valid_lft_, existing_lease->valid_lft_); + + } else { + // All other leases should have the same cltt. + EXPECT_EQ(leases6_[i]->cltt_, existing_lease->cltt_); + + // Leases with even indexes were added to the database with modified + // valid lifetime. Thus the local copy of each such lease should have + // this modified valid lifetime. The lease #0 should be updated from + // the partner because of the partner's cltt was set to later time. + if ((i != 0) && (i % 2) == 0) { + EXPECT_EQ(leases6_[i]->valid_lft_ - 1, existing_lease->valid_lft_); + + } else { + // All other leases should have been fetched from the partner and + // inserted with no change. + EXPECT_EQ(leases6_[i]->valid_lft_, existing_lease->valid_lft_); + } + } + } + } +} + +// Test that there is no exception thrown during IPv6 leases synchronization +// when server returns a wrong answer. +TEST_F(HAServiceTest, asyncSyncLeases6WrongAnswer) { + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false")); + + // Create IPv6 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases6()); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Set empty response. This should cause the HA service to log an + // error but not crash. + ElementPtr response_arguments = Element::createMap(); + + factory2_->getResponseCreator()->setArguments(response_arguments); + factory3_->getResponseCreator()->setArguments(response_arguments); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(1000)); +} + +// Test that there is no exception thrown during IPv6 leases synchronization +// when servers are offline. +TEST_F(HAServiceTest, asyncSyncLeases6ServerOffline) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + TestHAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service for 1 second. + ASSERT_NO_THROW(runIOService(1000)); +} + +// Test that there is no exception thrown during IPv6 leases synchronization +// when server requires not provided authentication. +TEST_F(HAServiceTest, asyncSyncLeases6Unauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create lease manager. + ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false")); + + // Create IPv6 leases which will be fetched from the other server. + ASSERT_NO_THROW(generateTestLeases6()); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Setting the heartbeat delay to 0 disables the recurring heartbeat. + // We just want to synchronize leases and not send the heartbeat. + config_storage->setHeartbeatDelay(0); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage, + HAServerType::DHCPv6); + + // Start fetching leases asynchronously. + ASSERT_NO_THROW(service.asyncSyncLeases()); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(1000)); +} + +// This test verifies that the ha-sync command is processed successfully for the +// DHCPv4 server. +TEST_F(HAServiceTest, processSynchronize4) { + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate success. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization" + " complete."); + + // All leases should have been inserted into the database. + for (size_t i = 0; i < leases4_.size(); ++i) { + Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText() + << " not in the lease database"; + } + + // The following commands should have been sent to the server2: dhcp-disable, + // lease4-get-page and ha-sync-complete-notify. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", "")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable", "")); +} + +// This test verifies that the ha-sync command is processed successfully for the +// DHCPv4 server. +TEST_F(HAServiceTest, processSynchronize4Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate success. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization" + " complete."); + + // All leases should have been inserted into the database. + for (size_t i = 0; i < leases4_.size(); ++i) { + Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText() + << " not in the lease database"; + } + + // The following commands should have been sent to the server2: dhcp-disable, + // lease4-get-page and ha-sync-complete-notify. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", "")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable", "")); +} + +// This test verifies that an error is reported when sending a dhcp-disable +// command causes an error. +TEST_F(HAServiceTest, processSynchronizeDisableError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("dhcp-disable", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should only receive dhcp-disable command. Remaining three should + // not be sent. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease4-get-page","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that an error is reported when sending any not +// authenticated command causes a not authorized error. +TEST_F(HAServiceTest, processSynchronizeUnauthorized) { + // Instruct server2 to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should only receive dhcp-disable commands. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); +} + +// This test verifies that an error is reported when sending a lease4-get-page +// command causes an error. +TEST_F(HAServiceTest, processSynchronizeLease4GetPageError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("lease4-get-page", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should receive dhcp-disable, lease4-get-page and dhcp-enable + // commands. The lease4-get-page command results in an error (configured at + // the beginning of this test), but the dhcp-enable command should be sent + // to enable the DHCP service after the error anyway. The + // ha-sync-complete-notify must not be sent after the error. It should only + // be sent after successful synchronization. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); +} + +// This test verifies that an error is reported when sending a dhcp-enable +// command causes an error. +TEST_F(HAServiceTest, processSynchronizeEnableError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("dhcp-enable", + CONTROL_RESULT_ERROR); + + // Return the unsupported command status for this command to enforce + // sending the dhcp-enable command. + factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify", + CONTROL_RESULT_COMMAND_UNSUPPORTED); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should receive four commands of which ha-sync-complete-notify + // was unsupported. + + auto requests = factory2_->getResponseCreator()->getReceivedRequests(); + ASSERT_GE(requests.size(), 3); + + // The dhcp-disable should be the first request. + auto request = factory2_->getResponseCreator()->findRequest("dhcp-disable","20"); + ASSERT_TRUE(request); + EXPECT_EQ(requests[0], request); + + // There may be multiple lease4-get-page commands but the first one should + // follow the dhcp-disable request. + request = factory2_->getResponseCreator()->findRequest("lease4-get-page", ""); + ASSERT_TRUE(request); + EXPECT_EQ(requests[1], request); + + // The ha-sync-complete-notify should be last but one. + request = factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", ""); + ASSERT_TRUE(request); + EXPECT_EQ(requests[requests.size()-2], request); + + // The dhcp-enable should be last. + request = factory2_->getResponseCreator()->findRequest("dhcp-enable", ""); + ASSERT_TRUE(request); + EXPECT_EQ(requests[requests.size()-1], request); +} + +// This test verifies that dhcp-enable command is not sent to the partner after +// receiving an error to the ha-sync-complete-notify command. +TEST_F(HAServiceTest, processSynchronizeNotifyError) { + // Return an error to the ha-sync-complete-notify command. + factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize4(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + auto requests = factory2_->getResponseCreator()->getReceivedRequests(); + + ASSERT_GE(requests.size(), 3); + + // The dhcp-disable should be the first request. + auto request = factory2_->getResponseCreator()->findRequest("dhcp-disable","20"); + ASSERT_TRUE(request); + EXPECT_EQ(requests[0], request); + + // There may be multiple lease4-get-page commands but the first one should + // follow the dhcp-disable request. + request = factory2_->getResponseCreator()->findRequest("lease4-get-page",""); + ASSERT_TRUE(request); + EXPECT_EQ(requests[1], request); + + // The ha-sync-complete-notify should be last. + request = factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""); + ASSERT_TRUE(request); + EXPECT_EQ(*(requests.rbegin()), request); + + // The dhcp-enable should not be sent after ha-sync-complete-notify failure. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that the ha-sync command is processed successfully for the +// DHCPv6 server. +TEST_F(HAServiceTest, processSynchronize6) { + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate success. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization" + " complete."); + + // All leases should have been inserted into the database. + for (size_t i = 0; i < leases6_.size(); ++i) { + Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText() + << " not in the lease database"; + } + + // The following commands should have been sent to the server2: dhcp-disable, + // lease6-get-page and ha-sync-complete-notify. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); +} + +// This test verifies that the ha-sync command is processed successfully for the +// DHCPv6 server. +TEST_F(HAServiceTest, processSynchronize6Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate success. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization" + " complete."); + + // All leases should have been inserted into the database. + for (size_t i = 0; i < leases6_.size(); ++i) { + Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA, + leases6_[i]->addr_); + ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText() + << " not in the lease database"; + } + + // The following commands should have been sent to the server2: dhcp-disable, + // lease6-get-page and ha-sync-complete-notify. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); +} + +// This test verifies that an error is reported when sending a dhcp-disable +// command causes an error. +TEST_F(HAServiceTest, processSynchronize6DisableError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("dhcp-disable", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should only receive dhcp-disable command. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that an error is reported when sending any not +// authenticated command causes a not authorized error. +TEST_F(HAServiceTest, processSynchronize6Unauthorized) { + // Instruct server2 to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should only receive dhcp-disable commands. + EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty()); +} + +// This test verifies that an error is reported when sending a lease6-get-page +// command causes an error. +TEST_F(HAServiceTest, processSynchronizeLease6GetPageError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("lease6-get-page", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should receive all commands. The dhcp-disable was successful, so + // the dhcp-enable command must be sent to re-enable the service after failure. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); +} + +// This test verifies that an error is reported when sending a dhcp-enable +// command causes an error. +TEST_F(HAServiceTest, processSynchronize6EnableError) { + // Setup the server2 to return an error to dhcp-disable commands. + factory2_->getResponseCreator()->setControlResult("dhcp-enable", + CONTROL_RESULT_ERROR); + + // Return the unsupported command status for this command to enforce + // sending the dhcp-enable command. + factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify", + CONTROL_RESULT_COMMAND_UNSUPPORTED); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + // The server2 should receive four commands of which ha-sync-complete-notify + // was unsupported. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that dhcp-enable command is not sent to the partner after +// receiving an error to the ha-sync-complete-notify command. +TEST_F(HAServiceTest, processSynchronize6NotifyError) { + // Return an error to the ha-sync-complete-notify command. + factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify", + CONTROL_RESULT_ERROR); + + // Run HAService::processSynchronize and gather a response. + ConstElementPtr rsp; + runProcessSynchronize6(rsp); + + // The response should indicate an error + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR); + + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page","")); + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","")); + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that the DHCPv4 service can be disabled on the remote server. +TEST_F(HAServiceTest, asyncDisableDHCPService4) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-disable command with max-period of 10 seconds. + // When the transaction is finished, the IO service gets stopped. + ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10, + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_TRUE(success); + EXPECT_TRUE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); + + // The second server should not receive the command. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-disable","10")); + // The third server should receive the dhcp-disable command with the max-period + // value of 10. + EXPECT_TRUE(factory3_->getResponseCreator()->findRequest("dhcp-disable","10")); +} + +// This test verifies that the DHCPv4 service can be disabled on the remote server. +TEST_F(HAServiceTest, asyncDisableDHCPService4Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-disable command with max-period of 10 seconds. + // When the transaction is finished, the IO service gets stopped. + ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10, + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_TRUE(success); + EXPECT_TRUE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); + + // The second server should not receive the command. + EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-disable","10")); + // The third server should receive the dhcp-disable command with the max-period + // value of 10. + EXPECT_TRUE(factory3_->getResponseCreator()->findRequest("dhcp-disable","10")); +} + +// This test verifies that there is no exception thrown as a result of dhcp-disable +// command when the server is offline. +TEST_F(HAServiceTest, asyncDisableDHCPService4ServerOffline) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-disable command with max-period of 10 seconds. + // When the transaction is finished, the IO service gets stopped. + ASSERT_NO_THROW(service.asyncDisableDHCPService("server2", 10, + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that an error is returned when the remote server +// returns control status error. +TEST_F(HAServiceTest, asyncDisableDHCPService4ControlResultError) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + // Set the servers to return error code in response to the dhcp-enable + // command. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-disable command with max-period of 10 seconds. + // When the transaction is finished, the IO service gets stopped. + ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10, + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that an error is returned when the remote server +// requires not provided authentication. +TEST_F(HAServiceTest, asyncDisableDHCPService4ControlResultUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-disable command with max-period of 10 seconds. + // When the transaction is finished, the IO service gets stopped. + ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10, + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that the DHCPv4 service can be enabled on the remote server. +TEST_F(HAServiceTest, asyncEnableDHCPService4) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-enable command. When the transaction is finished, + // the IO service gets stopped. + ASSERT_NO_THROW(service.asyncEnableDHCPService("server2", + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_TRUE(success); + EXPECT_TRUE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); + + // The second server should receive the dhcp-enable. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); + // The third server should not receive the command. + EXPECT_FALSE(factory3_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that the DHCPv4 service can be enabled on the remote server. +TEST_F(HAServiceTest, asyncEnableDHCPService4Authorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-enable command. When the transaction is finished, + // the IO service gets stopped. + ASSERT_NO_THROW(service.asyncEnableDHCPService("server2", + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_TRUE(success); + EXPECT_TRUE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); + + // The second server should receive the dhcp-enable. + EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable","")); + // The third server should not receive the command. + EXPECT_FALSE(factory3_->getResponseCreator()->findRequest("dhcp-enable","")); +} + +// This test verifies that there is no exception thrown as a result of dhcp-enable +// command when the server is offline. +TEST_F(HAServiceTest, asyncEnableDHCPService4ServerOffline) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-enable command. When the transaction is finished, + // the IO service gets stopped. + ASSERT_NO_THROW(service.asyncEnableDHCPService("server2", + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that an error is returned when the remote server +// returns control status error. +TEST_F(HAServiceTest, asyncEnableDHCPService4ControlResultError) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + // Set the servers to return error code in response to the dhcp-enable + // command. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-enable command. When the transaction is finished, + // the IO service gets stopped. + ASSERT_NO_THROW(service.asyncEnableDHCPService("server2", + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that an error is returned when the remote server +// requires not provided authentication. +TEST_F(HAServiceTest, asyncEnableDHCPService4ControlResultUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + listener3_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + // Send dhcp-enable command. When the transaction is finished, + // the IO service gets stopped. + ASSERT_NO_THROW(service.asyncEnableDHCPService("server2", + [this](const bool success, + const std::string& error_message, + const int) { + EXPECT_FALSE(success); + EXPECT_FALSE(error_message.empty()); + io_service_->stop(); + })); + + // Run IO service to actually perform the transaction. + ASSERT_NO_THROW(runIOService(TEST_TIMEOUT)); +} + +// This test verifies that the "ha-scopes" command is processed correctly. +TEST_F(HAServiceTest, processScopes) { + // Create HA configuration. + HAConfigPtr config_storage = createValidConfiguration(); + + // Create HA service using this configuration. + TestHAService service(io_service_, network_state_, config_storage); + + // Enable "server1" and "server2" scopes. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processScopes({ "server1", "server2" })); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured."); + + // Verify that "server1" and "server2" scopes are enabled. + EXPECT_TRUE(service.query_filter_.amServingScope("server1")); + EXPECT_TRUE(service.query_filter_.amServingScope("server2")); + EXPECT_FALSE(service.query_filter_.amServingScope("server3")); + + // Enable "server2" scope only. + ASSERT_NO_THROW(rsp = service.processScopes({ "server2" })); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured."); + EXPECT_FALSE(service.query_filter_.amServingScope("server1")); + EXPECT_TRUE(service.query_filter_.amServingScope("server2")); + EXPECT_FALSE(service.query_filter_.amServingScope("server3")); + + // Clear scopes. + ASSERT_NO_THROW(rsp = service.processScopes({ })); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured."); + EXPECT_FALSE(service.query_filter_.amServingScope("server1")); + EXPECT_FALSE(service.query_filter_.amServingScope("server2")); + EXPECT_FALSE(service.query_filter_.amServingScope("server3")); + + // Unsupported scope. + ASSERT_NO_THROW(rsp = service.processScopes({ "server1", "unsupported", "server3" })); + checkAnswer(rsp, CONTROL_RESULT_ERROR, "invalid server name specified 'unsupported'" + " while enabling/disabling HA scopes"); + // Even though the "server1" is a valid scope name, it should not be + // enabled because we expect scopes enabling to be atomic operation, + // i.e. all or nothing. + EXPECT_FALSE(service.query_filter_.amServingScope("server1")); + EXPECT_FALSE(service.query_filter_.amServingScope("server2")); + EXPECT_FALSE(service.query_filter_.amServingScope("server3")); +} + +// This test verifies that the ha-continue command is processed successfully. +TEST_F(HAServiceTest, processContinue) { + HAConfigPtr config_storage = createValidConfiguration(); + + // State machine is to be paused in the waiting state. + ASSERT_NO_THROW(config_storage->getStateMachineConfig()-> + getStateConfig(HA_WAITING_ST)->setPausing("always")); + + TestHAService service(io_service_, network_state_, config_storage); + + // Pause the state machine. + EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT)); + EXPECT_TRUE(service.isModelPaused()); + + // Process ha-continue command that should unpause the state machine. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processContinue()); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine continues."); + + // State machine should have been unpaused as a result of processing the + // command. + EXPECT_FALSE(service.isModelPaused()); + + // Response should include no arguments. + EXPECT_FALSE(rsp->get("arguments")); + + // Sending ha-continue command again which should return success but + // slightly different textual status. + ASSERT_NO_THROW(rsp = service.processContinue()); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine is not paused."); + + // The state machine should not be paused. + EXPECT_FALSE(service.isModelPaused()); + + // Response should include no arguments. + EXPECT_FALSE(rsp->get("arguments")); +} + +// This test verifies that the ha-maintenance-notify command is processed +// successfully and transitions the state machine to the in-maintenance state +// and that it is possible to revert to the previous state. +// It also verifies that this command fails to transition the state machine +// for some selected states for which it is unsupported. +TEST_F(HAServiceTest, processMaintenanceNotify) { + HAConfigPtr config_storage = createValidConfiguration(); + + std::set valid_states = { + HA_WAITING_ST, + HA_READY_ST, + HA_SYNCING_ST, + HA_PARTNER_DOWN_ST, + HA_LOAD_BALANCING_ST, + HA_HOT_STANDBY_ST, + }; + + TestHAService service(io_service_, network_state_, config_storage); + + // Test transition from the states for which it is allowed. + for (auto state : valid_states) { + EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT)); + + // Process ha-maintenance-notify command that should transition the + // state machine to the in-maintenance state. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false)); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."); + + // The state machine should have been transitioned to the in-maintenance state. + EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState()); + + // Try to cancel the maintenance. + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true)); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance canceled."); + + // The state machine should have been transitioned to the state it was in + // prior to transitioning to the in-maintenance state. + EXPECT_EQ(state, service.getCurrState()); + } + + std::set invalid_states = { + HA_TERMINATED_ST, + HA_BACKUP_ST, + HA_PARTNER_IN_MAINTENANCE_ST + }; + + // Make sure that the transition from the other states is not allowed. + for (auto state : invalid_states) { + EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT)); + EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT)); + + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false)); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED, + "Unable to transition the server from the " + + stateToString(state) + " to in-maintenance state."); + } +} + +// This test verifies the case when the server receiving the ha-maintenance-start +// command successfully transitions to the partner-in-maintenance state and its +// partner transitions to the in-maintenance state. +TEST_F(HAServiceTest, processMaintenanceStartSuccess) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner of our server is online and should have responded with + // the success status. Therefore, this server should have transitioned + // to the partner-in-maintenance state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is now in the partner-in-maintenance state" + " and its partner is in-maintenance state. The partner can be now safely" + " shut down."); + + EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState()); +} + +// This test verifies the case when the server receiving the ha-maintenance-start +// command successfully transitions to the partner-in-maintenance state and its +// partner transitions to the in-maintenance state. +TEST_F(HAServiceTest, processMaintenanceStartSuccessAuthorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner of our server is online and should have responded with + // the success status. Therefore, this server should have transitioned + // to the partner-in-maintenance state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is now in the partner-in-maintenance state" + " and its partner is in-maintenance state. The partner can be now safely" + " shut down."); + + EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState()); +} + +// This test verifies the case that the server transitions to the partner-down +// state after receiving the ha-maintenance-start command. This is the case +// when the communication with the partner server fails while this command +// is received. It is assumed that the partner server is already terminated +// for maintenance. +TEST_F(HAServiceTest, processMaintenanceStartPartnerDown) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Start the server, but don't start the partner. This simulates + // the case that the partner is already down for maintenance. + ASSERT_NO_THROW({ + listener_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner of our server is online and should have responded with + // the success status. Therefore, this server should have transitioned + // to the partner-in-maintenance state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, + "Server is now in the partner-down state as its" + " partner appears to be offline for maintenance."); + + EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState()); +} + +// This test verifies the case when the server is receiving +// ha-maintenance-start command and tries to notify the partner +// which returns an error. +TEST_F(HAServiceTest, processMaintenanceStartPartnerError) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Simulate an error returned by the partner. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, + "Server is now in the partner-down state as its" + " partner appears to be offline for maintenance."); + + EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState()); +} + +// This test verifies the case when the server is receiving +// ha-maintenance-start command and tries to notify the partner +// which requires not provided authentication. +TEST_F(HAServiceTest, processMaintenanceStartPartnerUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, + "Server is now in the partner-down state as its" + " partner appears to be offline for maintenance."); + + EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState()); +} + +// This test verifies the case when the server is receiving +// ha-maintenance-start command and tries to notify the partner +// which returns a special result indicating that it can't enter +// the in-maintenance state. +TEST_F(HAServiceTest, processMaintenanceStartNotAllowed) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Simulate an error returned by the partner. + factory2_->getResponseCreator()-> + setControlResult(TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + HAService service(io_service_, network_state_, config_storage); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-start command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceStart()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR, + "Unable to transition to the partner-in-maintenance state." + " The partner server responded with the following message" + " to the ha-maintenance-notify command: response returned" + " (error code 1001)."); + + // The partner's state precludes entering the in-maintenance state. Thus, this + // server must not change its state either. + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies the case when the server receiving the ha-maintenance-cancel +// command successfully transitions out of the partner-in-maintenance state. +TEST_F(HAServiceTest, processMaintenanceCancelSuccess) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner of our server is online and should have responded with + // the success status. Therefore, this server should have transitioned + // to the partner-in-maintenance state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled."); + + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies the case when the server receiving the ha-maintenance-cancel +// command successfully transitions out of the partner-in-maintenance state. +TEST_F(HAServiceTest, processMaintenanceCancelSuccessAuthorized) { + // Update config to provide authentication. + user2_ = "foo"; + password2_ = "bar"; + user3_ = "test"; + password3_ = "1234"; + + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner of our server is online and should have responded with + // the success status. Therefore, this server should have transitioned + // to the partner-in-maintenance state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled."); + + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies that the maintenance is not canceled in case the +// partner returns an error. +TEST_F(HAServiceTest, processMaintenanceCancelPartnerError) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Simulate an error returned by the partner. + factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner should have responded with an error. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR, + "Unable to cancel maintenance. The partner server responded" + " with the following message to the ha-maintenance-notify" + " command: response returned (error code 1)."); + + // The state of this server should not change. + EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState()); +} + +// This test verifies that the maintenance is not canceled in case the +// partner requires not provided authentication. +TEST_F(HAServiceTest, processMaintenanceCancelPartnerUnauthorized) { + // Instruct servers to require authentication. + factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); + factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(io_service_, network_state_, config_storage); + + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->get_io_service().reset(); + io_service_->poll(); + + // The partner should have responded with an error. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_ERROR, + "Unable to cancel maintenance. The partner server responded" + " with the following message to the ha-maintenance-notify" + " command: Unauthorized (error code 1)."); + + // The state of this server should not change. + EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState()); +} + +// This test verifies that the ha-reset command is processed successfully. +TEST_F(HAServiceTest, processHAReset) { + HAConfigPtr config_storage = createValidConfiguration(); + TestHAService service(io_service_, network_state_, config_storage); + + // Transition the server to the load-balancing state. + EXPECT_NO_THROW(service.transition(HA_LOAD_BALANCING_ST, HAService::NOP_EVT)); + + // Process ha-reset command that should cause the server to transition + // to the waiting state. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processHAReset()); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine reset."); + + // Response should include no arguments. + EXPECT_FALSE(rsp->get("arguments")); + + // The server should be in the waiting state. + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies that the ha-reset command is processed successfully when +// the server is already in the waiting state. +TEST_F(HAServiceTest, processHAResetWaiting) { + HAConfigPtr config_storage = createValidConfiguration(); + TestHAService service(io_service_, network_state_, config_storage); + + // Transition the server to the waiting state. + EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT)); + + // Process ha-reset command that should not change the state of the + // server because the server is already in the waiting state. It + // should not fail though. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processHAReset()); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."); + + // Response should include no arguments. + EXPECT_FALSE(rsp->get("arguments")); + + // The server should be in the waiting state. + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies that the ha-sync-complete-notify command is processed +// successfully, the server keeps the DHCP service disabled in the partner-down +// state and enables the service when it is in another state. +TEST_F(HAServiceTest, processSyncCompleteNotify) { + HAConfigPtr config_storage = createValidConfiguration(); + TestHAService service(io_service_, network_state_, config_storage); + + // Transition the server to the partner-down state. + EXPECT_NO_THROW(service.transition(HA_PARTNER_DOWN_ST, HAService::NOP_EVT)); + + // Simulate disabling the DHCP service for synchronization. + EXPECT_NO_THROW(service.network_state_->disableService(NetworkState::Origin::HA_COMMAND)); + + ConstElementPtr rsp; + EXPECT_NO_THROW(rsp = service.processSyncCompleteNotify()); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, + "Server successfully notified about the synchronization completion."); + + // The server should remain in the partner-down state. + EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState()); + + // The service should be disabled until the server transitions to the + // normal state. + EXPECT_FALSE(service.network_state_->isServiceEnabled()); + + // It is possible that the connection from this server to the partner + // is still broken. In that case, the HA_SYNCED_PARTNER_UNAVAILABLE_EVT + // is emitted and the server should enable DHCP service to continue + // serving the clients in the partner-down state. + EXPECT_NO_THROW(service.postNextEvent(HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT)); + EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT)); + EXPECT_TRUE(service.network_state_->isServiceEnabled()); + + // Transition the server to the load-balancing state. + EXPECT_NO_THROW(service.transition(HA_LOAD_BALANCING_ST, HAService::NOP_EVT)); + + // Disable the service again. + EXPECT_NO_THROW(service.network_state_->disableService(NetworkState::Origin::HA_COMMAND)); + + EXPECT_NO_THROW(rsp = service.processSyncCompleteNotify()); + + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, + "Server successfully notified about the synchronization completion."); + + // The server should remain in the load-balancing state. + EXPECT_EQ(HA_LOAD_BALANCING_ST, service.getCurrState()); + + // This time the service should be enabled because the server is in + // the state in which it sends the lease updates. + EXPECT_TRUE(service.network_state_->isServiceEnabled()); +} + +/// @brief HA partner to the server under test. +/// +/// This is a wrapper class around @c HttpListener which simulates a +/// partner server. It provides convenient methods to start, stop the +/// partner (its listener) and to transition the partner between various +/// HA states. Depending on the state and whether the partner is started +/// or stopped, different answers are returned in response to the +/// ha-heartbeat commands. +class HAPartner { +public: + + /// @brief Constructor. + /// + /// Creates the partner instance from a listner and the corresponding + /// response factory. It automatically transitions the partner to the + /// "waiting" state unless otherwise specified with the third parameter. + /// + /// @param listner pointer to the listner to be used. + /// @param factory pointer to the response factory to be used. This + /// must be the same factory that the listener is using. + /// @param initial_state initial state for the partner. Default is to + /// transition the partner to the "waiting" state which is the default + /// state for each starting server. + HAPartner(const HttpListenerPtr& listener, + const TestHttpResponseCreatorFactoryPtr& factory, + const std::string& initial_state = "waiting") + : listener_(listener), factory_(factory), running_(false), + static_date_time_(), static_scopes_(), + static_unsent_update_count_(0) { + transition(initial_state); + } + + /// @brief Sets control result to be returned as a result of the + /// communication with the partner. + /// + /// @param control_result new control result value. + void setControlResult(const int control_result) { + factory_->getResponseCreator()->setControlResult(control_result); + } + + /// @brief Sets static date-time value to be used in responses. + /// + /// @param static_date_time fixed date-time value. + void setDateTime(const std::string& static_date_time) { + static_date_time_ = static_date_time; + } + + /// @brief Sets static scopes to be used in responses. + /// + /// @param scopes Fixed scopes set. + void setScopes(const std::set& scopes) { + static_scopes_ = scopes; + } + + /// @brief Enable response to commands required for leases synchronization. + /// + /// Enables dhcp-disable, dhcp-enable and lease4-get-page commands. The last + /// of them returns a bunch of test leases. + void enableRespondLeaseFetching() { + // Create IPv4 leases which will be fetched from the other server. + std::vector leases4; + ASSERT_NO_THROW(generateTestLeases(leases4)); + + // Convert leases to the JSON format, the same as used by the lease_cmds + // hook library. Configure our test HTTP servers to return those + // leases in this format. + ElementPtr response_arguments = Element::createMap(); + response_arguments->set("leases", getLeasesAsJson(leases4)); + + factory_->getResponseCreator()->setArguments("lease4-get-page", response_arguments); + } + + /// @brief Starts up the partner. + void startup() { + if (!running_) { + listener_->start(); + running_ = true; + } + } + + /// @brief Shuts down the partner. + /// + /// It may be used to simulate partner's crash as well as graceful + /// shutdown. + void shutdown() { + if (running_) { + listener_->stop(); + running_ = false; + } + } + + /// @brief Transitions the partner to the specified state. + /// + /// The state is provided in the textual form and the function doesn't + /// validate whether it is correct or not. + /// + /// @param state new partner state. + void transition(const std::string& state) { + ElementPtr response_arguments = Element::createMap(); + response_arguments->set("state", Element::create(state)); + if (!static_date_time_.empty()) { + response_arguments->set("date-time", Element::create(static_date_time_)); + } + if (!static_scopes_.empty()) { + auto json_scopes = Element::createList(); + for (auto scope : static_scopes_) { + json_scopes->add(Element::create(scope)); + } + response_arguments->set("scopes", json_scopes); + } + if (static_unsent_update_count_ >= 0) { + response_arguments->set("unsent-update-count", Element::create(static_unsent_update_count_)); + } + factory_->getResponseCreator()->setArguments(response_arguments); + } + + /// @brief Sets static value of unsent update count. + /// + /// @param unsent_update_count new value of the unsent update count. + /// Specify a negative value to exclude the count from the response. + void setUnsentUpdateCount(int64_t unsent_update_count) { + static_unsent_update_count_ = unsent_update_count; + } + +private: + + /// @brief Instance of the listener wrapped by this class. + HttpListenerPtr listener_; + /// @brief Instance of the response factory used by the listener. + TestHttpResponseCreatorFactoryPtr factory_; + + /// @brief IPv4 leases to be used in the tests. + std::vector leases4_; + + /// @brief Boolean flag indicating if the partner is running. + bool running_; + + /// @brief Static date-time value to be returned. + std::string static_date_time_; + + /// @brief Static scopes to be reported. + std::set static_scopes_; + + /// @brief Static count of lease updates not sent by the partner + /// because the other server was unavailable. + int64_t static_unsent_update_count_; +}; + +/// @brief Shared pointer to a partner. +typedef boost::shared_ptr HAPartnerPtr; + +/// @brief Test fixture class for the HA service state machine. +class HAServiceStateMachineTest : public HAServiceTest { +public: + /// @brief Constructor. + HAServiceStateMachineTest() + : HAServiceTest(), state_(), + partner_(new HAPartner(listener2_, factory2_)) { + } + + /// @brief Creates common HA service instance from the provided configuration. + /// + /// The custom @c state_ object is created and it replaces the default + /// @c state_ object of the HA service. + /// + /// @param config pointer to the configuration to be used by the service. + /// @param server_type server type, i.e. DHCPv4 or DHCPv6. + void startService(const HAConfigPtr& config, + const HAServerType& server_type = HAServerType::DHCPv4) { + config->setHeartbeatDelay(1); + config->setSyncPageLimit(1000); + createSTService(network_state_, config, server_type); + // Replace default communication state with custom state which exposes + // protected members and methods. + state_.reset(new NakedCommunicationState4(io_service_, config)); + service_->communication_state_ = state_; + // Move the state machine from initial state to "waiting" state. + service_->runModel(HAService::NOP_EVT); + } + + /// @brief Runs IO service until specified event occurs. + /// + /// This method runs IO service until state machine is run as a result + /// of receiving a response to an IO operation. IO operations such as + /// lease updates, heartbeats etc. trigger state machine changes. + /// We can capture certain events to detect when a response to the heartbeat + /// or other control commands is received. This is useful to return control + /// to a test to verify that the state machine remains in the expected state + /// after receiving such response. + /// + /// @param event an event which should trigger IO service to stop. + void waitForEvent(const int event) { + ASSERT_NE(event, HAService::NOP_EVT); + + service_->postNextEvent(HAService::NOP_EVT); + + // Run IO service until the event occurs. + runIOService(TEST_TIMEOUT, [this, event]() { + return (service_->getLastEvent() == event); + }); + + service_->postNextEvent(HAService::NOP_EVT); + } + + /// @brief Convenience method checking if HA service is currently running + /// recurring heartbeat. + /// + /// @return true if the heartbeat is run. + bool isDoingHeartbeat() { + return (state_->isHeartbeatRunning()); + } + + /// @brief Convenience method checking if HA service has detected communications + /// interrupted condition. + /// + /// @return true if the communications interrupted condition deemed, false + /// otherwise. + bool isCommunicationInterrupted() { + return (state_->isCommunicationInterrupted()); + } + + /// @brief Convenience method checking if communication failure has been + /// detected by the HA service based on the analysis of the DHCP traffic. + /// + /// @return true if the communication failure is deemed, false otherwise. + bool isFailureDetected() { + return (state_->failureDetected()); + } + + /// @brief Simulates a case when communication with the partner has failed + /// for a time long enough to assume communications interrupted condition. + /// + /// This case is simulated by modifying the last poking time far into the + /// past. + void simulateNoCommunication() { + state_->modifyPokeTime(-1000); + } + + /// @brief Simulates reception of unanswered DHCP queries by the partner. + /// + /// This case is simulated by creating a large number of queries with + /// secs field set to high value. + void simulateDroppedQueries() { + // Create 100 packets. Around 50% of them should be assigned to the + // partner if load balancing is performed. + const unsigned queries_num = 100; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // Set large secs field value. + query4->setSecs(0x00EF); + // This function, besides checking if the query is in scope, + // updates unanswered message counters. If the counters exceed + // a configured value the communication failure is assumed. + static_cast(service_->inScope(query4)); + } + // The state machine needs to react to the dropped queries. Therefore + // we run the machine now. + service_->runModel(HAService::NOP_EVT); + } + + /// @brief Simulates rejected lease updates by the partner. + /// + /// Too many rejected lease updates should transition the server to the + /// terminated state. + /// @param leases_num number of simulated rejected leases. + void simulateRejectedLeaseUpdates(const unsigned leases_num = 100) { + // If the state machine is about to transition to another state, + // let's make sure it performs this transition. + service_->runModel(HAService::NOP_EVT); + // Simulate the rejected lease updates. + for (auto i = 0; i < leases_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + static_cast(state_->reportRejectedLeaseUpdate(query4, 100)); + } + // The state machine needs to react to the rejected leases. + service_->runModel(HAService::NOP_EVT); + } + + /// @brief Checks transitions dependent on the partner's state. + /// + /// This method uses @c partner_ object to control the state of the partner. + /// This method must not be used to test transitions from the syncing state + /// because this state includes synchronous IO operations. There is a + /// separate test for the transitions from the syncing state. + /// + /// @param my_state initial state of this server. + /// @param partner_state state of the partner. + /// @param final_state expected state to transition to. + void testTransition(const MyState& my_state, const PartnerState& partner_state, + const FinalState& final_state) { + // We need to shutdown the partner only if the partner is to be in the + // 'unavailable state'. + if (partner_state.state_ != HA_UNAVAILABLE_ST) { + // This function is not meant for testing transitions from the syncing + // state when partner is available. + ASSERT_NE(my_state.state_, HA_SYNCING_ST); + partner_->setControlResult(CONTROL_RESULT_SUCCESS); + + } else { + partner_->setControlResult(CONTROL_RESULT_ERROR); + } + + // Transition this server to the desired initial state. + service_->transition(my_state.state_, HAService::NOP_EVT); + // Transition the partner to the desired state. + partner_->transition(service_->getStateLabel(partner_state.state_)); + // Run the heartbeat. + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + // Make sure that this server ended up in the expected state. + EXPECT_EQ(final_state.state_, service_->getCurrState()) + << "expected transition to the '" + << service_->getStateLabel(final_state.state_) + << "' state for the partner state '" << service_->getStateLabel(partner_state.state_) + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + + // If the partner is unavailable we also have to verify the case when + // we detect that the partner is considered offline (after running the + // whole failure detection algorithm). The server in the in-maintenance + // state is excluded from this because it must not transition out of this + // state until an administrator makes some action. + if ((my_state.state_ != HA_IN_MAINTENANCE_ST) && + (partner_state.state_ == HA_UNAVAILABLE_ST)) { + // Transition this server back to the initial state. + service_->transition(my_state.state_, HAService::NOP_EVT); + // Simulate lack of communication between the servers. + simulateNoCommunication(); + // Send the heartbeat again. + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + + // The load balancing server or the standby server is monitoring the stream + // of packets directed to the partner to detect delays in responses. The + // primary server in the hot standby configuration doesn't do it, because + // the partner is not meant to process any queries until it detects that + // the primary server is down. This is only done in states in which the + // DHCP service is enabled. Otherwise, the server doesn't receive DHCP + // queries it could analyze. + if (service_->network_state_->isServiceEnabled() && + ((service_->config_->getHAMode() == HAConfig::LOAD_BALANCING) || + service_->config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY)) { + // The server should remain in its current state until we also detect + // that the partner is not answering the queries. + ASSERT_EQ(final_state.state_, service_->getCurrState()) + << "expected transition to the '" + << service_->getStateLabel(final_state.state_) + << "' state for the partner state '" << service_->getStateLabel(partner_state.state_) + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + + // Back to the original state again. + service_->transition(my_state.state_, HAService::NOP_EVT); + // This time simulate no responses from the partner to the DHCP clients' + // requests. This should cause the server to transition to the partner + // down state regardless of the initial state. + simulateDroppedQueries(); + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()) + << "expected transition to the 'partner-down' state, but transitioned" + " to the '" << service_->getStateLabel(service_->getCurrState()) + << "' state"; + + // The primary server in the hot-standby configuration should transition to + // the partner-down state when there is no communication with the partner + // over the control channel. + } else { + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()) + << "expected transition to the 'partner-down' state, but transitioned" + " to the '" << service_->getStateLabel(service_->getCurrState()) + << "' state"; + } + } + } + + /// @brief Checks transitions from the syncing state. + /// + /// This method uses @c partner_ object to control the state of the partner. + /// + /// @param final_state expected final server state. + void testSyncingTransition(const FinalState& final_state) { + // Transition to the syncing state. + service_->transition(HA_SYNCING_ST, HAService::NOP_EVT); + partner_->transition("ready"); + state_->stopHeartbeat(); + + testSynchronousCommands([this]() { + service_->runModel(HAService::NOP_EVT); + }); + + state_->stopHeartbeat(); + + EXPECT_EQ(final_state.state_, service_->getCurrState()) + << "expected transition to the '" + << service_->getStateLabel(final_state.state_) + << "' state, but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + } + + /// @brief Tests transition from any state to "terminated". + /// + /// @param my_state initial server state. + void testTerminateTransition(const MyState& my_state) { + // Set the partner's time way in the past so as the clock skew gets high. + partner_->setDateTime("Sun, 06 Nov 1994 08:49:37 GMT"); + partner_->transition("ready"); + // Transition this server to the desired initial state. + service_->transition(my_state.state_, HAService::NOP_EVT); + // Run the heartbeat. + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + // The server should get into terminated state because of the high + // clock skew. + EXPECT_EQ(HA_TERMINATED_ST, service_->getCurrState()) + << "expected transition to the 'terminated' state" + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + } + + /// @brief Tests transition between states when passive-backup mode is + /// in use. + /// + /// @param my_state initial server state. + /// @param final_state expected final state. + void testPassiveBackupTransition(const MyState& my_state, + const FinalState& final_state) { + service_->transition(my_state.state_, HAService::NOP_EVT); + service_->runModel(TestHAService::NOP_EVT); + + EXPECT_EQ(final_state.state_, service_->getCurrState()) + << "expected transition to the '" + << service_->getStateLabel(final_state.state_) + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + } + + /// @brief Test that the server is serving expected scopes while being in a + /// certain state. + /// + /// @param my_state state of the server. + /// @param scopes vector of scopes which the server is expected to handle in this + /// state. + /// @param dhcp_enabled Indicates whether DHCP service is expected to be enabled + /// or disabled in the given state. + /// @param event Event to be passed to the tested handler. + void expectScopes(const MyState& my_state, const std::vector& scopes, + const bool dhcp_enabled, const int event = TestHAService::NOP_EVT) { + + // If expecting no scopes, let's enable some scope to make sure that the + // code changes this setting. + if (scopes.empty()) { + service_->query_filter_.serveScope("server1"); + + } else { + // If expecting some scopes, let's initially clear the scopes to make + // sure that the code sets it. + service_->query_filter_.serveNoScopes(); + } + + // Also, let's preset the DHCP server state to the opposite of the expected + // state. + if (dhcp_enabled) { + service_->network_state_->disableService(NetworkState::Origin::HA_COMMAND); + + } else { + service_->network_state_->enableService(NetworkState::Origin::HA_COMMAND); + } + + // Transition to the desired state. + service_->postNextEvent(event); + service_->verboseTransition(my_state.state_); + // Run the handler. + service_->runModel(TestHAService::NOP_EVT); + // First, check that the number of handlded scope is equal to the number of + // scopes specified as an argument. + ASSERT_EQ(scopes.size(), service_->query_filter_.getServedScopes().size()) + << "test failed for state '" << service_->getStateLabel(my_state.state_) + << "'"; + + // Now, verify that each specified scope is handled. + for(auto scope : scopes) { + EXPECT_TRUE(service_->query_filter_.amServingScope(scope)) + << "test failed for state '" << service_->getStateLabel(my_state.state_) + << "'"; + } + // Verify if the DHCP service is enabled or disabled. + EXPECT_EQ(dhcp_enabled, service_->network_state_->isServiceEnabled()) + << "test failed for state '" << service_->getStateLabel(my_state.state_) + << "'"; + } + + /// @brief Transitions the server to the specified state and checks if the + /// HA service would send lease updates in this state. + /// + /// @param my_state this server's state + /// @param peer_config configuration of the server to which lease updates are + /// to be sent. + /// @return true if the lease updates would be sent, false otherwise. + bool expectLeaseUpdates(const MyState& my_state, + const HAConfig::PeerConfigPtr& peer_config) { + service_->verboseTransition(my_state.state_); + return (service_->shouldSendLeaseUpdates(peer_config)); + } + + /// @brief Transitions the server to the specified state and checks if the + /// HA service would queue lease updates in this state. + /// + /// @param my_state this server's state + /// @param peer_config configuration of the server to which lease updates are + /// to be sent or queued. + /// @return true if the lease updates would be queued, false otherwise. + bool expectQueueLeaseUpdates(const MyState& my_state, + const HAConfig::PeerConfigPtr& peer_config) { + service_->verboseTransition(my_state.state_); + return (service_->shouldQueueLeaseUpdates(peer_config)); + } + + /// @brief Transitions the server to the specified state and checks if the + /// HA service is sending heartbeat in this state. + /// + /// @param my_state this server's state + /// @return true if the heartbeat is sent in this state, false otherwise. + bool expectHeartbeat(const MyState& my_state) { + service_->verboseTransition(my_state.state_); + service_->runModel(TestHAService::NOP_EVT); + return (isDoingHeartbeat()); + } + + /// @brief Transitions the server to the specified state and checks that it + /// clears the rejected leases counter. + /// + /// @param my_state this server's state + /// @return true if the rejected leases counter has been cleared. + bool expectClearRejectedLeases(const MyState& my_state) { + simulateRejectedLeaseUpdates(1); + service_->verboseTransition(my_state.state_); + service_->runModel(TestHAService::NOP_EVT); + return (state_->getRejectedLeaseUpdatesCount() == 0); + } + + /// @brief Pointer to the communication state used in the tests. + NakedCommunicationState4Ptr state_; + /// @brief Pointer to the partner used in some tests. + HAPartnerPtr partner_; +}; + + +// Test the following scenario: +// 1. I show up in waiting state and look around +// 2. My partner doesn't respond over control channel +// 3. I start analyzing partner's packets and see that +// it doesn't respond. +// 4. I transition to partner down state. +// 5. Partner finally shows up and eventually transitions to the ready state. +// 6. I see the partner being ready, so I synchronize myself and eventually +// transition to the load-balancing state. +// 7. Next, the partner crashes again. +// 8. I detect partner's crash and transition back to partner down. +// 9. While being in the partner down state, I find that the partner +// is available and it is doing load balancing. +// 10. I stay in the partner-down state to force the partner to transition +// to the waiting state and synchronize its database. +TEST_F(HAServiceStateMachineTest, waitingParterDownLoadBalancingPartnerDown) { + HAConfigPtr config_storage = createValidConfiguration(); + // Disable syncing leases to avoid transitions via the syncing state. + // In this state it is necessary to perform synchronous tasks. + config_storage->setSyncLeases(false); + startService(config_storage); + + // Start the server: offline ---> WAITING state. + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: no heartbeat response for a long period of time. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + simulateNoCommunication(); + ASSERT_TRUE(isDoingHeartbeat()); + + // WAITING state: communication interrupted. In this state we don't analyze + // packets ('secs' field) because the DHCP service is disabled. + // WAITING ---> PARTNER DOWN + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + + // PARTNER DOWN state: still no response from the partner. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + + // Partner shows up and (eventually) transitions to READY state. + HAPartner partner(listener2_, factory2_); + partner.setScopes({ "server1", "server2" }); + partner.setUnsentUpdateCount(10); + partner.transition("ready"); + partner.startup(); + + // PARTNER DOWN state: receive a response from the partner indicating that + // the partner is in READY state. + // PARTNER DOWN ---> WAITING + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // WAITING state: synchronization is disabled to simplify this test. The + // server should transition straight to the ready state. + // WAITING --> READY + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // READY --> LOAD BALANCING + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // Ensure that the state handler is invoked. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + + // Check the reported info about servers. + ConstElementPtr ha_servers = service_->processStatusGet(); + ASSERT_TRUE(ha_servers); + + // Hard to know what is the age of the remote data. Therefore, let's simply + // grab it from the response. + ASSERT_EQ(Element::map, ha_servers->getType()); + auto remote = ha_servers->get("remote"); + ASSERT_TRUE(remote); + EXPECT_EQ(Element::map, remote->getType()); + auto age = remote->get("age"); + ASSERT_TRUE(age); + EXPECT_EQ(Element::integer, age->getType()); + auto age_value = age->intValue(); + EXPECT_GE(age_value, 0); + + // Now append it to the whole structure for comparison. + std::ostringstream s; + s << age_value; + + std::string expected = "{" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ \"server1\" ], " + " \"state\": \"load-balancing\"" + " }, " + " \"remote\": {" + " \"age\": " + s.str() + "," + " \"in-touch\": true," + " \"role\": \"secondary\"," + " \"last-scopes\": [ \"server1\", \"server2\" ]," + " \"last-state\": \"ready\"," + " \"communication-interrupted\": false," + " \"connecting-clients\": 0," + " \"unacked-clients\": 0," + " \"unacked-clients-left\": 0," + " \"analyzed-packets\": 0" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + + // Crash the partner and see whether our server can return to the partner + // down state. + partner.setControlResult(CONTROL_RESULT_ERROR); + + // LOAD BALANCING state: wait for the next heartbeat to occur. This heartbeat + // fails causing the server to enter communication-recovery state in which the + // server is collecting lease updates to be sent when the communication is + // resumed. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // COMMUNICATION RECOVERY state: simulate lack of communication for a longer + // period of time. We should remain the communication-recovery state and + // keep analyzing the traffic directed to partner. + simulateNoCommunication(); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_TRUE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // COMMUNICATION RECOVERY state: simulate a lot of unanswered DHCP + // messages to the partner. This server should detect that the partner is + // not answering and transition to partner down state. + // COMMUNICATION RECOVERY ---> PARTNER DOWN + simulateDroppedQueries(); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_TRUE(isCommunicationInterrupted()); + ASSERT_TRUE(isFailureDetected()); + + // Start the partner again and transition it to the load balancing state. + partner.setControlResult(CONTROL_RESULT_SUCCESS); + partner.transition("load-balancing"); + + // PARTNER DOWN state: the partner shows up in the load-balancing state. + // It may happen when the partner did not crash but there was a temporary + // communication error with it. It is possible that this server was not + // configured to monitor unacked clients and that's why it transitioned + // to the partner-down state. The partner may be configured differently. + // The partner was not receiving lease updates from us, so we need to + // force it to transition to the waiting state and synchronize. We stay + // in the partner-down state as long as necessary to force the partner + // to synchronize. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); +} + +// Test the following scenario: +// 1. I show up in waiting state and look around. +// 2. My partner is offline and is not responding over the control channel. +// 3. I can't communicate with the partner so I transition to the partner-down +// state. +// 4. Partner shows up and eventually transitions to the ready state. +// 5. I see the partner being ready, so I synchronize myself and eventually +// transition to the load-balancing state. +// 6. Partner stops responding again. +// 7. I monitor communication with the partner and eventually consider the +// communication to be interrupted. +// 8. I start monitoring the DHCP traffic directed to the partner and observe +// delays in responses. +// 9. I transition to the partner-down state again seeing that the certain +// number of clients can't communicate with the partner. +// 10. The partner unexpectedly shows up in the hot-standby mode. I stay in +// the partner-down state to force the partner to transition to the waiting +// state and synchronize its database. +TEST_F(HAServiceStateMachineTest, waitingParterDownHotStandbyPartnerDown) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + // Disable syncing leases to avoid transitions via the syncing state. + // In this state it is necessary to perform synchronous tasks. + valid_config->setSyncLeases(false); + + // Start the server: offline ---> WAITING state. + startService(valid_config); + + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: no heartbeat response for a long period of time. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + simulateNoCommunication(); + ASSERT_TRUE(isDoingHeartbeat()); + + // WAITING state: communication interrupted. In this state we don't analyze + // packets ('secs' field) because the DHCP service is disabled. + // WAITING ---> PARTNER DOWN + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + + // PARTNER DOWN state: still no response from the partner. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + + // Partner shows up and (eventually) transitions to READY state. + partner_.reset(new HAPartner(listener_, factory_)); + partner_->setUnsentUpdateCount(10); + partner_->transition("ready"); + partner_->startup(); + + // PARTNER DOWN state: receive a response from the partner indicating that + // the partner is in READY state. + // PARTNER DOWN ---> WAITING + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // WAITING state: synchronization is disabled to simplify this test. The + // server should transition straight to the ready state. + // WAITING --> READY + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // Primary server must transition to the hot-standby state first before + // standby can transition. + partner_->transition("hot-standby"); + + // READY --> HOT STANDBY + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // Crash the partner and see whether our server can return to the partner + // down state. + partner_->setControlResult(CONTROL_RESULT_ERROR); + + // HOT STANDBY state: wait for the next heartbeat to occur and make + // sure that a single heartbeat loss is not yet causing us to assume + // partner down condition. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // HOT STANDBY state: simulate lack of communication for a longer + // period of time. We should remain in the hot-standby state waiting for + // unanswered DHCP traffic before going to partner-down state. + simulateNoCommunication(); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_TRUE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); + + // HOT STANDBY state: simulate a lot of unanswered DHCP + // messages to the partner. This server should detect that the partner is + // not answering and transition to partner down state. + // HOT STANDBY ---> PARTNER DOWN + simulateDroppedQueries(); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_TRUE(isCommunicationInterrupted()); + ASSERT_TRUE(isFailureDetected()); + + // Start the partner again and transition it to the hot standby state. + partner_->setControlResult(CONTROL_RESULT_SUCCESS); + partner_->transition("hot-standby"); + + // PARTNER DOWN state: the partner shows up in the hot-standby state. + // It may happen when the partner did not crash but there was a temporary + // communication error with it. The partner was not receiving lease updates + // from us, so we need to force it to transition to the waiting state and + // synchronize. We stay in the partner-down state as long as necessary to + // force the partner to synchronize. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); + ASSERT_TRUE(isDoingHeartbeat()); + ASSERT_FALSE(isCommunicationInterrupted()); + ASSERT_FALSE(isFailureDetected()); +} + +// Test the following scenario: +// 1. I begin in a load-balancing state. +// 2. My partner is offline. +// 3. I transition to the communication-recovery state. +// 4. My partner shows up in the load-balancing state. +// 5. I see the partner so I get back to load-balancing state as well. +TEST_F(HAServiceStateMachineTest, loadBalancingCommRecoveryLoadBalancing) { + startService(createValidConfiguration()); + service_->verboseTransition(HA_LOAD_BALANCING_ST); + service_->runModel(HAService::NOP_EVT); + + // Simulate that the partner is not responding to my queries. + simulateNoCommunication(); + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); + + HAPartner partner(listener2_, factory2_, "load-balancing"); + partner.startup(); + + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); +} + +// Test the following scenario: +// 1. I show up in the waiting state. +// 2. My partner appears to be in the partner-down state. +// 3. I proceed to the syncing state to fetch leases from the partner. +// 4. The first attempt to fetch leases is unsuccessful. +// 5. I remain in the syncing state until I am finally successful. +// 6. I proceed to the ready state. +// 7. I see that the partner is still in partner-down state, so I +// wait for the partner to transition to load-balancing state. +TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancing) { + // Partner is present and is in the PARTNER DOWN state. + HAPartner partner(listener2_, factory2_, "partner-down"); + partner.startup(); + + // Start the server: offline ---> WAITING state. + startService(createValidConfiguration()); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: receive a response from the partner indicating that the + // partner is in the load balancing state. I should transition to the + // SYNCING state to fetch leases from the partner. + // WAITING ---> SYNCING state + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState()); + + // We better stop the heartbeat to not interfere with the synchronous + // commands. + state_->stopHeartbeat(); + + // The database synchronization is synchronous operation so we need to run + // the partner's IO service in thread (in background). + testSynchronousCommands([this, &partner]() { + + // SYNCING state: the partner is up but it won't respond to the lease4-get-page + // command correctly. This should leave us in the SYNCING state until we finally + // can synchronize. + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState()); + + // We better stop the heartbeat to not interfere with the synchronous + // commands. + state_->stopHeartbeat(); + + // Enable the partner to correctly respond to the lease fetching and retry. + // We should successfully update the database and transition. + // SYNCING ---> READY + partner.enableRespondLeaseFetching(); + + // After previous attempt to synchronize the recorded partner state became + // "unavailable". This server won't synchronize until the heartbeat is + // sent which would indicate that the server is running. Therefore, we + // manually set the state of the partner to "partner-down". + state_->setPartnerState("partner-down"); + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + }); + + // READY state: I do another heartbeat but my partner still seems to be in the + // partner down state, so I can't transition to load balancing just yet. I + // still remain the READY state. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + + // READY state: partner transitions to the LOAD BALANCING state seeing that I + // am ready. I should see this transition and also transition to that state. + // READY ---> LOAD BALANCING. + partner.transition("load-balancing"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); +} + +// Test the following scenario: +// 1. I am a primary server which is started at the same time as secondary. +// 2. I determine that we're the primary server so I go ahead and start +// synchronizing. +// 3. Partner should also determine that it is a secondary server so it should +// remain in the waiting state until we're ready. +// 4. I synchronize the database and transition to the ready state. +// 5. The partner should see that I am ready and should start the transition. +// 6. I remain ready until the partner gets to the ready state. +// 7. I transition to the load balancing state when the partner is ready. +// 8. The partner transitions to the load balancing state which doesn't +// affect my state. +TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancingPrimary) { + // Partner is present and is in the WAITING state. + HAPartner partner(listener2_, factory2_); + partner.enableRespondLeaseFetching(); + partner.startup(); + + // Start the server: offline ---> WAITING state. + startService(createValidConfiguration()); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: both servers are in this state but our server is primary, + // so it transitions to the syncing state. The peer remains in the WAITING + // state until we're ready. + // WAITING --->SYNCING + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState()); + + // We better stop the heartbeat to not interfere with the synchronous + // commands. + state_->stopHeartbeat(); + + // SYNCING state: this server will synchronously fetch leases from the peer. + // Therefore, we need to run the IO service in the thread to allow for + // synchronous operations to complete. Once the leases are fetched it should + // transition to the READY state. + // SYNCING ---> READY. + testSynchronousCommands([this]() { + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + }); + + // READY state: our partner sees that we're ready so it will start to + // synchronize. We remain in the READY state as long as the partner is not + // ready. + partner.transition("syncing"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + + // READY state: our partner appears to be ready. We can now start load + // balancing. + // READY ---> LOAD BALANCING. + partner.transition("ready"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); + + // LOAD BALANCING state: our partner should eventually transition to the + // LOAD BALANCING state. This should not affect us doing load balancing. + partner.transition("load-balancing"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); +} + +// Test the following scenario: +// 1. I am secondary server configured for load balancing and started at the +// same time as the primary server. +// 2. I determine that I am a secondary server, so I remain in the waiting +// state until the primary server indicates that it is ready. +// 3. I start synchronizing my lease database when the partner is ready. +// 4. I transition to the ready state and leave in that state until I see +// the primary transition to the load balancing state. +// 5. I also transition to the load balancing state at that point. +TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancingSecondary) { + // Partner is present and is in the WAITING state. + HAPartner partner(listener_, factory_); + partner.enableRespondLeaseFetching(); + partner.startup(); + + // Create the configuration in which we're the secondary server doing + // load balancing. + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setThisServerName("server2"); + + // Start the server: offline ---> WAITING state. + startService(valid_config); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: our partner is a primary so we remain in the WAITING state + // until it indicates it is ready. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: the partner is synchronizing the lease database. We still + // remain in the WAITING state. + partner.transition("syncing"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()); + + // WAITING state: partner transitions to the READY state. This is a signal + // for us to start synchronization of the lease database. + // WAITING ---> SYNCING. + partner.transition("ready"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState()); + + // We better stop the heartbeat to not interfere with the synchronous + // commands. + state_->stopHeartbeat(); + + // SYNCING state: this server will synchronously fetch leases from the peer. + // Therefore, we need to run the IO service in the thread to allow for + // synchronous operations to complete. Once the leases are fetched it should + // transition to the READY state. + // SYNCING ---> READY. + testSynchronousCommands([this]() { + service_->runModel(HAService::NOP_EVT); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + }); + + // READY state: we remain ready until the other server transitions to the + // LOAD BALANCING state. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_READY_ST, service_->getCurrState()); + + // READY state: our primary server transitions to the LOAD BALANCING state. + // We can now also transition to this state. + // READY ---> LOAD BALANCING. + partner.transition("load-balancing"); + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState()); +} + +// This test checks all combinations of server and partner states and the +// resulting state to which the server transitions. This server is primary. +// There is another test which validates state transitions from the +// secondary server perspective. +TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) { + partner_->startup(); + + startService(createValidConfiguration()); + + // COMMUNICATION RECOVERY state transitions + { + SCOPED_TRACE("COMMUNICATION RECOVERY state transitions"); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_READY_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + } + + // LOAD BALANCING state transitions + { + SCOPED_TRACE("LOAD BALANCING state transitions"); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + } + + // in-maintenance state transitions + { + SCOPED_TRACE("in-maintenance state transitions"); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + } + + // PARTNER DOWN state transitions + { + SCOPED_TRACE("PARTNER DOWN state transitions"); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // PARTNER in-maintenance state transitions + { + SCOPED_TRACE("PARTNER in-maintenance state transitions"); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // READY state transitions + { + SCOPED_TRACE("READY state transitions"); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_READY_ST)); + } + + // WAITING state transitions + { + SCOPED_TRACE("WAITING state transitions"); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_WAITING_ST)); + } +} + +// This test checks that the server in the load balancing mode does not +// transition to the "syncing" state when "sync-leases" is disabled. +TEST_F(HAServiceStateMachineTest, noSyncingTransitionsLoadBalancingPrimary) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setSyncLeases(false); + startService(valid_config); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); +} + +// This test verifies that the load balancing server does not transition to +// the communication recovery state when delayed-updates-limit is set +// to 0. +TEST_F(HAServiceStateMachineTest, noCommunicationRecoverytransitionsLoadBalancingPrimary) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setDelayedUpdatesLimit(0); + startService(valid_config); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_LOAD_BALANCING_ST)); +} + +// This test checks that the server in the load balancing mode transitions to +// the "terminated" state when the clock skew gets high. +TEST_F(HAServiceStateMachineTest, terminateTransitionsLoadBalancingPrimary) { + partner_->startup(); + + startService(createValidConfiguration()); + + testTerminateTransition(MyState(HA_LOAD_BALANCING_ST)); + testTerminateTransition(MyState(HA_PARTNER_DOWN_ST)); + testTerminateTransition(MyState(HA_READY_ST)); + testTerminateTransition(MyState(HA_WAITING_ST)); +} + +// This test checks that the server does not transition out of the waiting state +// to the terminated state when the server is restarted but the clock skew has +// been corrected. +TEST_F(HAServiceStateMachineTest, terminateNoTransitionOnRestart) { + partner_->startup(); + startService(createValidConfiguration()); + + // Set partner's time to the current time. This guarantees that the clock + // skew is below 60s and there is no reason for the server to transition + // to the terminated state. + partner_->setDateTime(HttpDateTime().rfc1123Format()); + // The partner is in the terminated state to simulate sequential restart + // of the two servers from the terminated state. + partner_->transition("terminated"); + // This server is in the waiting state which simulates the restart case. + service_->transition(HA_WAITING_ST, HAService::NOP_EVT); + // Run the heartbeat. + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + // The server should remain in the waiting state because the clock skew + // is low. + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()) + << "expected that the server remains in 'waiting' state" + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; + + // Now, let's set the partner's time way to the past to verify that this + // server transitions to the 'terminated' state if the administrator + // failed to sync the clocks prior to the restart. + partner_->setDateTime("Sun, 06 Nov 1994 08:49:37 GMT"); + // Run the heartbeat. + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + EXPECT_EQ(HA_WAITING_ST, service_->getCurrState()) + << "expected that the server transitions to the 'terminated' state" + << "', but transitioned to the '" + << service_->getStateLabel(service_->getCurrState()) + << "' state"; +} + +// This test checks all combinations of server and partner states and the +// resulting state to which the server transitions. This server is secondary. +// There is another test which validates state transitions from the +// primary server perspective. +TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setThisServerName("server2"); + startService(valid_config); + + partner_->startup(); + + // COMMUNICATION RECOVERY state transitions + { + SCOPED_TRACE("COMMUNICATION RECOVERY state transitions"); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_READY_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + + testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + } + + // LOAD BALANCING state transitions + { + SCOPED_TRACE("LOAD BALANCING state transitions"); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_COMMUNICATION_RECOVERY_ST)); + } + + // in-maintenance state transitions + { + SCOPED_TRACE("in-maintenance state transitions"); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + } + + // PARTNER DOWN state transitions + { + SCOPED_TRACE("PARTNER DOWN state transitions"); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // PARTNER in-maintenance state transitions + { + SCOPED_TRACE("PARTNER in-maintenance state transitions"); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // READY state transitions + { + SCOPED_TRACE("READY state transitions"); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_READY_ST)); + } + + // WAITING state transitions + { + SCOPED_TRACE("WAITING state transitions"); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_WAITING_ST)); + } +} + +// This test checks that the server in the load balancing mode does not +// transition to the "syncing" state when "sync-leases" is disabled. +// This is the secondary server case. +TEST_F(HAServiceStateMachineTest, noSyncingTransitionsLoadBalancingSecondary) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setThisServerName("server2"); + valid_config->setSyncLeases(false); + startService(valid_config); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); +} + +// This test checks that the secondary server in the load balancing mode +// transitions to the "terminated" state when the clock skew gets high. +TEST_F(HAServiceStateMachineTest, terminateTransitionsLoadBalancingSecondary) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setThisServerName("server2"); + startService(valid_config); + + testTerminateTransition(MyState(HA_LOAD_BALANCING_ST)); + testTerminateTransition(MyState(HA_PARTNER_DOWN_ST)); + testTerminateTransition(MyState(HA_READY_ST)); + testTerminateTransition(MyState(HA_WAITING_ST)); +} + +// This test verifies that the backup server transitions to its own state. +TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingBackup) { + HAConfigPtr valid_config = createValidConfiguration(); + + // server3 is marked as a backup server. + valid_config->setThisServerName("server3"); + startService(valid_config); + + // The server should transition to the backup state and stay there. + for (unsigned i = 0; i < 10; ++i) { + service_->runModel(HAService::NOP_EVT); + ASSERT_EQ(HA_BACKUP_ST, service_->getCurrState()); + // In the backup state the DHCP service is disabled by default. + // It can only be enabled manually. + ASSERT_FALSE(service_->network_state_->isServiceEnabled()); + ASSERT_EQ(0, service_->query_filter_.getServedScopes().size()); + } +} + +// This test verifies transitions from the syncing state in the load +// balancing configuration. +TEST_F(HAServiceStateMachineTest, syncingTransitionsLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + startService(valid_config); + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + + // The syncing state handler doesn't start synchronization until it + // detects that the partner is online. It may remember that from the + // previous heartbeat attempts. If the partner appears to be unavailable + // it will continue heartbeats before it synchronizes. This prevents the + // server from making endless attempts to synchronize without any chance + // to succeed. We verify that the server is not trying to synchronize + // by checking that the last event is not the one associated with the + // synchronization attempt. + ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT); + ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT); + + // Run the syncing state handler. + testSyncingTransition(FinalState(HA_SYNCING_ST)); + + // We should see no synchronization attempts because the partner is + // offline. + EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT); + EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT); + + // Startup the partner. + partner_->enableRespondLeaseFetching(); + partner_->startup(); + + // We haven't been running heartbeats so we have to manually set the + // partner's state to something other than 'unavailable'. + state_->setPartnerState("ready"); + + // Retry the test. + testSyncingTransition(FinalState(HA_READY_ST)); + // This time the server should have synchronized. + EXPECT_EQ(HAService::HA_SYNCING_SUCCEEDED_EVT, service_->getLastEvent()); +} + +// This test verifies that the HA state machine can be paused in certain states +// when the server is operating in load balancing mode. The test also verifies +// that heartbeat is active even if the state machine is paused. +TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPause) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(); + auto state_machine = valid_config->getStateMachineConfig(); + + // Set state machine pausing in various states. + state_machine->getStateConfig(HA_LOAD_BALANCING_ST)->setPausing("always"); + state_machine->getStateConfig(HA_PARTNER_DOWN_ST)->setPausing("always"); + state_machine->getStateConfig(HA_READY_ST)->setPausing("always"); + state_machine->getStateConfig(HA_SYNCING_ST)->setPausing("always"); + state_machine->getStateConfig(HA_TERMINATED_ST)->setPausing("always"); + state_machine->getStateConfig(HA_WAITING_ST)->setPausing("always"); + + startService(valid_config); + + { + SCOPED_TRACE("LOAD BALANCING state transitions"); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_LOAD_BALANCING_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + + EXPECT_TRUE(service_->unpause()); + // An additional attempt to unpause should return false. + EXPECT_FALSE(service_->unpause()); + + testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + } + + { + SCOPED_TRACE("PARTNER DOWN state transitions"); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_PARTNER_DOWN_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + + EXPECT_TRUE(service_->unpause()); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_LOAD_BALANCING_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + } + + + { + SCOPED_TRACE("READY state transitions"); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_READY_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + + EXPECT_TRUE(service_->unpause()); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_LOAD_BALANCING_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + } + + + { + SCOPED_TRACE("WAITING state transitions"); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + + EXPECT_TRUE(service_->unpause()); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_SYNCING_ST)); + EXPECT_TRUE(state_->isHeartbeatRunning()); + } +} + +// This test verifies that the HA state machine can be paused in the syncing +// state. +TEST_F(HAServiceStateMachineTest, syncingTransitionsLoadBalancingPause) { + HAConfigPtr valid_config = createValidConfiguration(); + + auto state_machine = valid_config->getStateMachineConfig(); + + // Pause state machine in various states. + state_machine->getStateConfig(HA_LOAD_BALANCING_ST)->setPausing("always"); + state_machine->getStateConfig(HA_PARTNER_DOWN_ST)->setPausing("always"); + state_machine->getStateConfig(HA_READY_ST)->setPausing("always"); + state_machine->getStateConfig(HA_SYNCING_ST)->setPausing("always"); + state_machine->getStateConfig(HA_TERMINATED_ST)->setPausing("always"); + state_machine->getStateConfig(HA_WAITING_ST)->setPausing("always"); + + startService(valid_config); + waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT); + + // The syncing state handler doesn't start synchronization until it + // detects that the partner is online. It may remember that from the + // previous heartbeat attempts. If the partner appears to be unavailable + // it will continue heartbeats before it synchronizes. This prevents the + // server from making endless attempts to synchronize without any chance + // to succeed. We verify that the server is not trying to synchronize + // by checking that the last event is not the one associated with the + // synchronization attempt. + ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT); + ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT); + + // Startup the partner. + partner_->enableRespondLeaseFetching(); + partner_->startup(); + + // We haven't been running heartbeats so we have to manually set the + // partner's state to something other than 'unavailable'. + state_->setPartnerState("ready"); + + // Run the syncing state handler. + testSyncingTransition(FinalState(HA_SYNCING_ST)); + + // We should see no synchronization attempts because the server is paused + // in this state. + EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT); + EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT); + + // Unpause the state machine. + EXPECT_TRUE(service_->unpause()); + + // Retry the test. It should now transition to the ready state. + testSyncingTransition(FinalState(HA_READY_ST)); + + // This time the server should have synchronized. + EXPECT_EQ(HAService::HA_SYNCING_SUCCEEDED_EVT, service_->getLastEvent()); +} + +// This test verifies that the server takes ownership of the given scopes +// and whether the DHCP service is disabled or enabled in certain states. +TEST_F(HAServiceStateMachineTest, scopesServingLoadBalancing) { + startService(createValidConfiguration()); + + // LOAD BALANCING and TERMINATED: serving my own scope. + expectScopes(MyState(HA_LOAD_BALANCING_ST), { "server1" }, true); + expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true); + + // PARTNER DOWN and PARTNER IN MAINTENANCE: serving both scopes. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1", "server2" }, true); + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1", "server2" }, true); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies that the server does not take ownership of the +// partner's scope when auto-failover parameter is set to false. +TEST_F(HAServiceStateMachineTest, scopesServingLoadBalancingNoFailover) { + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->getThisServerConfig()->setAutoFailover(false); + startService(valid_config); + + // LOAD BALANCING and TERMINATED: serving my own scope. + expectScopes(MyState(HA_LOAD_BALANCING_ST), { "server1" }, true); + expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true); + + // PARTNER DOWN: still serving my own scope because auto-failover is disabled. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true); + + // PARTNER IN MAINTENANCE: always serving all scopes. + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1", "server2" }, true); + + // Same for the partner-down case during maintenance. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1", "server2" }, true, + HAService::HA_MAINTENANCE_START_EVT); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies if the server would send lease updates to the partner +// while being in various states. The HA configuration is load balancing. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config)); + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// Check that lease updates are sent to the backup server even when the +// secondary is in the partner-down state. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesToBackup) { + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setWaitBackupAck(false); + startService(valid_config); + + // Send the updates to the backup server. + HAConfig::PeerConfigPtr backup_config = valid_config->getPeerConfig("server3"); + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), backup_config)); +} + +// This test verifies if the server would not send lease updates to the +// partner if lease updates are administratively disabled. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesDisabledLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setSendLeaseUpdates(false); + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// Check that lease updates to the backup server are not queued. +TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesToBackup) { + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setWaitBackupAck(false); + startService(valid_config); + + HAConfig::PeerConfigPtr backup_config = valid_config->getPeerConfig("server3"); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), backup_config)); +} + +// This test verifies if the server would queue lease updates to the partner +// while being in communication-recovery state. The HA configuration is load +// balancing. +TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_TRUE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// This test verifies if the server would not queue lease updates to the +// partner if lease updates are administratively disabled. +TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesDisabledLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + valid_config->setSendLeaseUpdates(false); + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// This test verifies if the server would send heartbeat to the partner +// while being in various states. The HA configuration is load balancing. +TEST_F(HAServiceStateMachineTest, heartbeatLoadBalancing) { + HAConfigPtr valid_config = createValidConfiguration(); + startService(valid_config); + + EXPECT_TRUE(expectHeartbeat(MyState(HA_LOAD_BALANCING_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_DOWN_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_READY_ST))); + EXPECT_FALSE(expectHeartbeat(MyState(HA_TERMINATED_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_WAITING_ST))); +} + +// This test checks all combinations of server and partner states and the +// resulting state to which the server transitions. This server is primary. +// There is another test which validates state transitions from the +// standby server perspective. +TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + // HOT STANDBY state transitions + { + SCOPED_TRACE("HOT STANDBY state transitions"); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_READY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_HOT_STANDBY_ST)); + } + + // in-maintenance state transitions + { + SCOPED_TRACE("in-maintenance state transitions"); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + } + + // PARTNER DOWN state transitions + { + SCOPED_TRACE("PARTNER DOWN state transitions"); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // PARTNER in-maintenance state transitions + { + SCOPED_TRACE("PARTNER in-maintenance state transitions"); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + // READY state transitions + { + SCOPED_TRACE("READY state transitions"); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_READY_ST)); + } + + // WAITING state transitions + { + SCOPED_TRACE("WAITING state transitions"); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_WAITING_ST)); + } +} + +// This test checks that the server in the hot standby mode does not +// transition to the "syncing" state when "sync-leases" is disabled. +// This is the primary server case. +TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyPrimary) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->getPeerConfig("server2")->setRole("standby"); + valid_config->setSyncLeases(false); + + startService(valid_config); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); +} + +// This test checks that the primary server in the hot standby mode +// transitions to the "terminated" state when the clock skew gets high. +TEST_F(HAServiceStateMachineTest, terminateTransitionsHotStandbyPrimary) { + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + testTerminateTransition(MyState(HA_HOT_STANDBY_ST)); + testTerminateTransition(MyState(HA_PARTNER_DOWN_ST)); + testTerminateTransition(MyState(HA_READY_ST)); + testTerminateTransition(MyState(HA_WAITING_ST)); +} + +// This test checks all combinations of server and partner states and the +// resulting state to which the server transitions. This server is standby. +// There is another test which validates state transitions from the +// primary server perspective. +TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + // HOT STANDBY state transitions + { + SCOPED_TRACE("HOT STANDBY state transitions"); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_READY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_HOT_STANDBY_ST)); + } + + // in-maintenance state transitions + { + SCOPED_TRACE("in-maintenance state transitions"); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + } + + // PARTNER DOWN state transitions + { + SCOPED_TRACE("PARTNER DOWN state transitions"); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_PARTNER_DOWN_ST)); + + testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_PARTNER_DOWN_ST)); + } + + + // READY state transitions + { + SCOPED_TRACE("READY state transitions"); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_HOT_STANDBY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_PARTNER_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_IN_MAINTENANCE_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_TERMINATED_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_READY_ST)); + } + + // WAITING state transitions + { + SCOPED_TRACE("WAITING state transitions"); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_SYNCING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST), + FinalState(HA_WAITING_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST), + FinalState(HA_WAITING_ST)); + } +} + +// This test checks that the server in the hot standby mode does not +// transition to the "syncing" state when "sync-leases" is disabled. +// This is the standby server case. +TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyStandby) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + valid_config->setSyncLeases(false); + + startService(valid_config); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST), + FinalState(HA_READY_ST)); + + testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST), + FinalState(HA_READY_ST)); +} + +// This test checks that the standby server in the hot standby mode +// transitions to the "terminated" state when the clock skew gets high. +TEST_F(HAServiceStateMachineTest, terminateTransitionsHotStandbyStandby) { + partner_.reset(new HAPartner(listener_, factory_)); + partner_->startup(); + + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + + // Turn it into hot-standby configuration. + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + testTerminateTransition(MyState(HA_HOT_STANDBY_ST)); + testTerminateTransition(MyState(HA_PARTNER_DOWN_ST)); + testTerminateTransition(MyState(HA_READY_ST)); + testTerminateTransition(MyState(HA_WAITING_ST)); +} + +// This test verifies that the server takes ownership of the given scopes +// and whether the DHCP service is disabled or enabled in certain states. +// This is primary server. +TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyPrimary) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + // HOT STANDBY and TERMINATED: serving my own scope + expectScopes(MyState(HA_HOT_STANDBY_ST), { "server1" }, true); + expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true); + + // PARTNER DOWN and PARTNER IN MAINTENANCE: still serving my own scope. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true); + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies that auto-failover setting does not affect scopes +// handling by the primary server in the hot-standby mode. +TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyPrimaryNoFailover) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->getPeerConfig("server2")->setRole("standby"); + + // Disable auto-failover. + valid_config->getThisServerConfig()->setAutoFailover(false); + + startService(valid_config); + + // HOT STANDBY and TERMINATED: serving my own scope + expectScopes(MyState(HA_HOT_STANDBY_ST), { "server1" }, true); + expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true); + + // PARTNER IN MAINTENANCE & PARTNER DOWN: still serving my own scope. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true); + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies if the server would send lease updates to the partner +// while being in various states. The HA configuration is hot standby and +// the server is primary. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesHotStandbyPrimary) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_HOT_STANDBY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// This test verifies if the server would send heartbeat to the partner +// while being in various states. The HA configuration is hot standby. +TEST_F(HAServiceStateMachineTest, heartbeatHotStandby) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + EXPECT_TRUE(expectHeartbeat(MyState(HA_HOT_STANDBY_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_DOWN_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_READY_ST))); + EXPECT_FALSE(expectHeartbeat(MyState(HA_TERMINATED_ST))); + EXPECT_TRUE(expectHeartbeat(MyState(HA_WAITING_ST))); +} + +// This test verifies that the server takes ownership of the given scopes +// and whether the DHCP service is disabled or enabled in certain states. +// This is standby server. +TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyStandby) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + // HOT STANDBY: serving no scopes because the primary is active. + // The DHCP is enabled to be able to monitor activity of the + // primary. + expectScopes(MyState(HA_HOT_STANDBY_ST), { }, true); + + // TERMINATED: serving no scopes because the primary is active. + expectScopes(MyState(HA_TERMINATED_ST), { }, true); + + // PARTNER IN MAINTENANCE & PARTNER DOWN: serving server1's scope. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true); + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies that the standby server does not take ownership +// of the primary server's scope when auto-failover is set to false +TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyStandbyNoFailover) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + // Disable auto-failover. + valid_config->getThisServerConfig()->setAutoFailover(false); + + startService(valid_config); + + // HOT STANDBY: serving no scopes because the primary is active. + // The DHCP is enabled to be able to monitor activity of the + // primary. + expectScopes(MyState(HA_HOT_STANDBY_ST), { }, true); + + // TERMINATED: serving no scopes because the primary is active. + expectScopes(MyState(HA_TERMINATED_ST), { }, true); + + // PARTNER DOWN: still serving no scopes because auto-failover is set to false. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { }, true); + + // PARTNER IN MAINTENANCE: serving partner's scopes. + expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true); + + // Same for the partner-down case during maintenance. + expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true, + HAService::HA_MAINTENANCE_START_EVT); + + // IN MAINTENANCE, READY & WAITING: serving no scopes. + expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false); + expectScopes(MyState(HA_READY_ST), { }, false); + expectScopes(MyState(HA_WAITING_ST), { }, false); +} + +// This test verifies if the server would send lease updates to the partner +// while being in various states. The HA configuration is hot standby and +// the server is secondary. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesHotStandbyStandby) { + HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY); + valid_config->setThisServerName("server2"); + valid_config->getPeerConfig("server2")->setRole("standby"); + + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig(); + + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_HOT_STANDBY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config)); + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// This test verifies if the backup server doesn't send lease updates. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesBackup) { + HAConfigPtr valid_config = createValidConfiguration(); + + // Turn it into hot-standby configuration. + valid_config->setThisServerName("server3"); + + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getPeerConfig("server1"); + + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_BACKUP_ST), peer_config)); + EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// This test checks all combinations of the primary server in the passive +// backup configuration. +TEST_F(HAServiceStateMachineTest, stateTransitionsPassiveBackup) { + partner_->startup(); + + HAConfigPtr valid_config = createValidPassiveBackupConfiguration(); + startService(valid_config); + + testPassiveBackupTransition(MyState(HA_WAITING_ST), FinalState(HA_PASSIVE_BACKUP_ST)); + testPassiveBackupTransition(MyState(HA_PASSIVE_BACKUP_ST), FinalState(HA_PASSIVE_BACKUP_ST)); +} + +// This test checks that no scopes are being served in the waiting state +// of the passive backup configuration and that active server's scope is +// served while in the passive-backup state. +TEST_F(HAServiceStateMachineTest, scopesServingPassiveBackup) { + HAConfigPtr valid_config = createValidPassiveBackupConfiguration(); + startService(valid_config); + + EXPECT_TRUE(service_->query_filter_.amServingScope("server1")); + EXPECT_FALSE(service_->query_filter_.amServingScope("server2")); + EXPECT_TRUE(service_->network_state_->isServiceEnabled()); +} + +// Verifies that lease updates are sent by the server in the passive-backup state. +TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesPassiveBackup) { + HAConfigPtr valid_config = createValidPassiveBackupConfiguration(); + startService(valid_config); + + HAConfig::PeerConfigPtr peer_config = valid_config->getPeerConfig("server2"); + + EXPECT_TRUE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config)); +} + +// Verifies that the server transitions to the terminated state as a result +// of too many rejected leases. +TEST_F(HAServiceStateMachineTest, shouldTerminateDueToRejectedLeases) { + startService(createValidConfiguration()); + service_->verboseTransition(HA_LOAD_BALANCING_ST); + service_->runModel(HAService::NOP_EVT); + + // Simulate many rejected leases. It should cause the server to transition + // to the terminated state. + simulateRejectedLeaseUpdates(100); + EXPECT_EQ(HA_TERMINATED_ST, service_->getCurrState()); +} + +// Verifies that rejected leases are cleared upon entering certain states and +// are not cleared upon entering other states in the load balancing mode. +TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesLoadBalancing) { + startService(createValidConfiguration()); + + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_COMMUNICATION_RECOVERY_ST))); + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_LOAD_BALANCING_ST))); + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_PARTNER_DOWN_ST))); + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PARTNER_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_READY_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_TERMINATED_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_WAITING_ST))); +} + +// Verifies that rejected leases are cleared upon entering certain states and +// are not cleared upon entering other states in the hot standby mode. +TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesHotStandby) { + HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY); + config_storage->getPeerConfig("server2")->setRole("standby"); + startService(config_storage); + + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_HOT_STANDBY_ST))); + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_PARTNER_DOWN_ST))); + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PARTNER_IN_MAINTENANCE_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_READY_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_TERMINATED_ST))); + EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_WAITING_ST))); +} + +// Verifies that rejected leases are cleared upon entering certain states and +// are not cleared upon entering other states in the passive backup mode. +TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesPassiveBackup) { + HAConfigPtr config_storage = createValidPassiveBackupConfiguration(); + config_storage->getPeerConfig("server2")->setRole("backup"); + startService(config_storage); + + EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PASSIVE_BACKUP_ST))); +} + +// Verifies that the server doesn't terminate when the partner is unavailable. +TEST_F(HAServiceStateMachineTest, doNotTerminateWhenPartnerUnavailable) { + HAConfigPtr config_storage = createValidConfiguration(); + ASSERT_NO_THROW_LOG(service_.reset(new TestHAService(io_service_, network_state_, + config_storage))); + NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_, + config_storage)); + service_->communication_state_ = state; + + // Begin in the load-balancing state and the partner is also in + // that state. + service_->verboseTransition(HA_LOAD_BALANCING_ST); + service_->runModel(HAService::NOP_EVT); + service_->communication_state_->poke(); + service_->communication_state_->setPartnerState("load-balancing"); + + // Change the clock skew, so it is beyond the acceptable threshold. + state->clock_skew_ += boost::posix_time::time_duration(0, 1, 5); + + // Send the heartbeat. We didn't setup the connection to the partner. + // The heartbeat should fail and our server should transition to the + // communication-recovery state. Earlier, the server had a bug that + // it would take the last known clock skew and transition to the + // terminated state instead. It was wrong because when the partner is + // unavailable we can't tell the current clock skew. The administrator + // may actually be fixing the time on the other server. Thus, we must + // not terminate. + ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT)); + EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState()); +} + +} // end of anonymous namespace diff --git a/src/hooks/dhcp/high_availability/tests/ha_test.cc b/src/hooks/dhcp/high_availability/tests/ha_test.cc new file mode 100644 index 0000000..7c90242 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_test.cc @@ -0,0 +1,378 @@ +// Copyright (C) 2017-2023 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isc::asiolink; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::hooks; + +namespace { + +/// @brief Structure that holds registered hook indexes. +struct TestHooks { + int hooks_index_dhcp4_srv_configured_; + + /// Constructor that registers hook points for the tests. + TestHooks() { + hooks_index_dhcp4_srv_configured_ = HooksManager::registerHook("dhcp4_srv_configured"); + } +}; + +TestHooks Hooks; + +} + +namespace isc { +namespace ha { +namespace test { + +HATest::HATest() + : io_service_(new IOService()), + network_state_(new NetworkState(NetworkState::DHCPv4)) { +} + +HATest::~HATest() { +} + +void +HATest::startHAService() { + if (HooksManager::calloutsPresent(Hooks.hooks_index_dhcp4_srv_configured_)) { + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("io_context", io_service_); + callout_handle->setArgument("network_state", network_state_); + HooksManager::callCallouts(Hooks.hooks_index_dhcp4_srv_configured_, + *callout_handle); + } +} + +void +HATest::runIOService(long ms) { + io_service_->get_io_service().reset(); + IntervalTimer timer(*io_service_); + timer.setup(std::bind(&IOService::stop, io_service_), ms, + IntervalTimer::ONE_SHOT); + io_service_->run(); + timer.cancel(); +} + +void +HATest::runIOService(long ms, std::function stop_condition) { + io_service_->get_io_service().reset(); + IntervalTimer timer(*io_service_); + bool timeout = false; + timer.setup(std::bind(&HATest::stopIOServiceHandler, this, std::ref(timeout)), + ms, IntervalTimer::ONE_SHOT); + + while (!stop_condition() && !timeout) { + io_service_->run_one(); + } + + timer.cancel(); +} + +boost::shared_ptr +HATest::runIOServiceInThread() { + io_service_->get_io_service().reset(); + + bool running = false; + std::mutex mutex; + std::condition_variable condvar; + + io_service_->post(std::bind(&HATest::signalServiceRunning, this, std::ref(running), + std::ref(mutex), std::ref(condvar))); + boost::shared_ptr + th(new std::thread(std::bind(&IOService::run, io_service_.get()))); + + std::unique_lock lock(mutex); + while (!running) { + condvar.wait(lock); + } + + return (th); +} + +void +HATest::testSynchronousCommands(std::function commands) { + // Run IO service in thread. + auto thread = runIOServiceInThread(); + + // Run desired commands. + commands(); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); +} + +void +HATest::signalServiceRunning(bool& running, std::mutex& mutex, + std::condition_variable& condvar) { + std::lock_guard lock(mutex); + running = true; + condvar.notify_one(); +} + +void +HATest::stopIOServiceHandler(bool& stop_flag) { + stop_flag = true; +} + +ConstElementPtr +HATest::createValidJsonConfiguration(const HAConfig::HAMode& ha_mode) const { + std::ostringstream config_text; + config_text << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": " << (ha_mode == HAConfig::LOAD_BALANCING ? + "\"load-balancing\"" : "\"hot-standby\"") << "," + " \"sync-page-limit\": 3," + " \"heartbeat-delay\": 1000," + " \"max-response-delay\": 1000," + " \"max-ack-delay\": 10000," + " \"max-unacked-clients\": 10," + " \"max-rejected-clients\": 10," + " \"multi-threading\": {" + " \"enable-multi-threading\": false" + " }," + " \"wait-backup-ack\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:18123/\"," + " \"role\": \"primary\"," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:18124/\"," + " \"role\": " << (ha_mode == HAConfig::LOAD_BALANCING ? + "\"secondary\"" : "\"standby\"") << "," + " \"auto-failover\": true" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:18125/\"," + " \"role\": \"backup\"," + " \"auto-failover\": false" + " }" + " ]" + " }" + "]"; + + return (Element::fromJSON(config_text.str())); +} + +ConstElementPtr +HATest::createValidPassiveBackupJsonConfiguration() const { + std::ostringstream config_text; + config_text << + "[" + " {" + " \"this-server-name\": \"server1\"," + " \"mode\": \"passive-backup\"," + " \"multi-threading\": {" + " \"enable-multi-threading\": false" + " }," + " \"wait-backup-ack\": false," + " \"peers\": [" + " {" + " \"name\": \"server1\"," + " \"url\": \"http://127.0.0.1:18123/\"," + " \"role\": \"primary\"" + " }," + " {" + " \"name\": \"server2\"," + " \"url\": \"http://127.0.0.1:18124/\"," + " \"role\": \"backup\"" + " }," + " {" + " \"name\": \"server3\"," + " \"url\": \"http://127.0.0.1:18125/\"," + " \"role\": \"backup\"" + " }" + " ]" + " }" + "]"; + + return (Element::fromJSON(config_text.str())); +} + + +HAConfigPtr +HATest::createValidConfiguration(const HAConfig::HAMode& ha_mode) const { + HAConfigPtr config_storage(new HAConfig()); + HAConfigParser parser; + + parser.parse(config_storage, createValidJsonConfiguration(ha_mode)); + return (config_storage); +} + +HAConfigPtr +HATest::createValidPassiveBackupConfiguration() const { + HAConfigPtr config_storage(new HAConfig()); + HAConfigParser parser; + + parser.parse(config_storage, createValidPassiveBackupJsonConfiguration()); + return (config_storage); +} + +void +HATest::checkAnswer(const isc::data::ConstElementPtr& answer, + const int exp_status, + const std::string& exp_txt) { + int rcode = 0; + isc::data::ConstElementPtr comment; + comment = isc::config::parseAnswer(rcode, answer); + + EXPECT_EQ(exp_status, rcode) + << "Expected status code " << exp_status + << " but received " << rcode << ", comment: " + << (comment ? comment->str() : "(none)"); + + // Ok, parseAnswer interface is weird. If there are no arguments, + // it returns content of text. But if there is an argument, + // it returns the argument and it's not possible to retrieve + // "text" (i.e. comment). + if (comment->getType() != Element::string) { + comment = answer->get("text"); + } + + if (!exp_txt.empty()) { + EXPECT_EQ(exp_txt, comment->stringValue()); + } +} + +std::vector +HATest::randomKey(const size_t key_size) const { + std::vector key(key_size); + util::fillRandom(key.begin(), key.end()); + return (key); +} + + +Pkt4Ptr +HATest::createMessage4(const uint8_t msg_type, const uint8_t hw_address_seed, + const uint8_t client_id_seed, const uint16_t secs) const { + Pkt4Ptr message(new Pkt4(msg_type, 0x1234)); + + HWAddrPtr hw_address(new HWAddr(std::vector(6, hw_address_seed), + HTYPE_ETHER)); + message->setHWAddr(hw_address); + message->setSecs(secs); + + if (client_id_seed > 0) { + OptionPtr opt_client_id(new Option(Option::V4, DHO_DHCP_CLIENT_IDENTIFIER, + std::vector(6, client_id_seed))); + message->addOption(opt_client_id); + } + + return (message); +} + +Pkt4Ptr +HATest::createQuery4(const std::string& hw_address_text) const { + Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 0x1234)); + HWAddr hwaddr = HWAddr::fromText(hw_address_text); + query4->setHWAddr(HWAddrPtr(new HWAddr(hwaddr.hwaddr_, HTYPE_ETHER))); + return (query4); +} + +Pkt4Ptr +HATest::createQuery4(const std::vector& hw_address, + const std::vector& client_id) const { + Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 0x1234)); + query4->setHWAddr(HWAddrPtr(new HWAddr(hw_address, HTYPE_ETHER))); + if (!client_id.empty()) { + OptionPtr opt_client_id(new Option(Option::V4, DHO_DHCP_CLIENT_IDENTIFIER, + client_id)); + query4->addOption(opt_client_id); + } + return (query4); +} + +Pkt6Ptr +HATest::createQuery6(const std::vector& duid) const { + Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 0x1234)); + OptionPtr opt_duid(new Option(Option::V6, D6O_CLIENTID, duid)); + query6->addOption(opt_duid); + return (query6); +} + +Pkt6Ptr +HATest::createMessage6(const uint8_t msg_type, const uint8_t duid_seed, + const uint16_t elapsed_time) const { + Pkt6Ptr message(new Pkt6(msg_type, 0x1234)); + + OptionPtr opt_duid(new Option(Option::V6, D6O_CLIENTID, OptionBuffer(8, duid_seed))); + message->addOption(opt_duid); + + OptionUint16Ptr opt_elapsed_time(new OptionUint16(Option::V6, D6O_ELAPSED_TIME, + elapsed_time)); + message->addOption(opt_elapsed_time); + + return (message); +} + +Pkt6Ptr +HATest::createQuery6(const std::string& duid_text) const { + Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 0x1234)); + DUID duid = DUID::fromText(duid_text); + OptionPtr client_id(new Option(Option::V6, D6O_CLIENTID, duid.getDuid())); + query6->addOption(client_id); + return (query6); +} + +void +HATest::setDHCPMultiThreadingConfig(bool enabled, uint32_t thread_pool_size, + uint32_t packet_queue_size) { + ElementPtr mt_config = Element::createMap(); + mt_config->set("enable-multi-threading", Element::create(enabled)); + mt_config->set("thread-pool-size", + Element::create(static_cast(thread_pool_size))); + mt_config->set("queue-size", + Element::create(static_cast(packet_queue_size))); + CfgMgr::instance().getStagingCfg()->setDHCPMultiThreading(mt_config); +} + +std::string +HATest::makeHAMtJson(bool enable_multi_threading, bool http_dedicated_listener, + uint32_t http_listener_threads, uint32_t http_client_threads) { + std::stringstream ss; + ss << "\"multi-threading\": {" + << " \"enable-multi-threading\": " + << (enable_multi_threading ? "true" : "false") << "," + << " \"http-dedicated-listener\": " + << (http_dedicated_listener ? "true" : "false") << "," + << " \"http-listener-threads\": " << http_listener_threads << "," + << " \"http-client-threads\": " << http_client_threads << "}"; + return (ss.str()); +} + +} // namespace test +} // namespace ha +} // namespace isc diff --git a/src/hooks/dhcp/high_availability/tests/ha_test.h b/src/hooks/dhcp/high_availability/tests/ha_test.h new file mode 100644 index 0000000..0b763ca --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/ha_test.h @@ -0,0 +1,304 @@ +// Copyright (C) 2017-2022 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace isc { +namespace ha { +namespace test { + +/// @brief Derivation of the @c CommunicationState class which allows +/// for modifications of poke time. +/// +/// @tparam @c CommunicationState4 or @c CommunicationState6. +template +class NakedCommunicationState : public StateType { +public: + + /// @brief Constructor. + /// + /// @param io_service pointer to the IO service object. + explicit NakedCommunicationState(const asiolink::IOServicePtr& io_service, + const HAConfigPtr& config) + : StateType(io_service, config) { + } + + /// @brief Checks if the object was poked recently. + /// + /// @return true if the object was poked less than 5 seconds ago, + /// false otherwise. + bool isPoked() const { + return (StateType::getDurationInMillisecs() < 5000); + } + + using StateType::config_; + using StateType::timer_; + using StateType::clock_skew_; + using StateType::last_clock_skew_warn_; + using StateType::my_time_at_skew_; + using StateType::partner_time_at_skew_; + using StateType::unsent_update_count_; + using StateType::getRejectedLeaseUpdatesCountFromContainer; +}; + +/// @brief Type of the NakedCommunicationState for DHCPv4. +typedef NakedCommunicationState NakedCommunicationState4; + +/// @brief Type of the pointer to the @c NakedCommunicationState4. +typedef boost::shared_ptr NakedCommunicationState4Ptr; + +/// @brief Type of the NakedCommunicationState for DHCPv6. +typedef NakedCommunicationState NakedCommunicationState6; + +/// @brief Type of the pointer to the @c NakedCommunicationState6. +typedef boost::shared_ptr NakedCommunicationState6Ptr; + +/// @brief General test fixture class for all HA unittests. +/// +/// It provides basic functions to load and unload HA hooks library. +/// All test classes should derive from this class. +class HATest : public ::testing::Test { +public: + + /// @brief Constructor. + HATest(); + + /// @brief Destructor. + virtual ~HATest(); + + /// @brief Calls dhcp4_srv_configured callout to set IO service pointer. + void startHAService(); + + /// @brief Runs IO service for a specified amount of time. + /// + /// @param ms number of milliseconds for which the IO service should be + /// run. + void runIOService(long ms); + + /// @brief Runs IO service until timeout occurs or until provided method + /// returns true. + /// + /// @param ms number of milliseconds for which the IO service should be + /// run. + /// @param stop_condition pointer to the function which returns true if + /// when the IO service should be stopped. + void runIOService(long ms, std::function stop_condition); + + /// @brief Runs IO service in a thread. + /// + /// @return Shared pointer to the thread. + boost::shared_ptr runIOServiceInThread(); + + /// @brief Executes commands while running IO service in a thread. + /// + /// @param commands pointer to a function to be executed while IO service + /// is run in thread. + void testSynchronousCommands(std::function commands); + +protected: + + /// @brief Signals that the IO service is running. + /// + /// @param running reference to the flag which is set to true when the + /// IO service starts running and executes this function. + /// @param mutex reference to the mutex used for synchronization. + /// @param condvar reference to condition variable used for synchronization. + void signalServiceRunning(bool& running, std::mutex& mutex, + std::condition_variable& condvar); + +public: + + /// @brief Handler for timeout during runIOService invocation. + /// + /// @param [out] stop_flag set to true when the handler is invoked. + void stopIOServiceHandler(bool& stop_flag); + + /// @brief Return HA configuration with three servers in JSON format. + /// + /// @param ha_mode HA operation mode (default is load balancing). + /// @return Pointer to the unparsed configuration. + data::ConstElementPtr + createValidJsonConfiguration(const HAConfig::HAMode& ha_mode = + HAConfig::LOAD_BALANCING) const; + + /// @brief Return HA configuration with one primary and two backup + /// servers in the JSON format. + /// + /// @return Pointer to the unparsed configuration. + data::ConstElementPtr + createValidPassiveBackupJsonConfiguration() const; + + /// @brief Return HA configuration with three servers. + /// + /// @param ha_mode HA operation mode (default is load balancing). + /// @return Pointer to the parsed configuration. + HAConfigPtr createValidConfiguration(const HAConfig::HAMode& ha_mode = + HAConfig::LOAD_BALANCING) const; + + /// @brief Return passive-backup configuration. + /// + /// @return Pointer to the parsed configuration. + HAConfigPtr createValidPassiveBackupConfiguration() const; + + /// @brief Checks the status code and message against expected values. + /// + /// @param answer Element set containing an integer response and string + /// comment. + /// @param exp_status Status code against which to compare the status. + /// @param exp_txt Expected text (not checked if empty) + void checkAnswer(const isc::data::ConstElementPtr& answer, + const int exp_status, + const std::string& exp_txt = ""); + + /// @brief Creates an identifier of arbitrary size with random values. + /// + /// This function is useful in generating random client identifiers and + /// HW addresses for load balancing tests. + /// + /// @param key_size Size of the generated random identifier. + std::vector randomKey(const size_t key_size) const; + + /// @brief Generates simple DHCPv4 message. + /// + /// @param msg_type DHCPv4 message type to be created. + /// @param hw_address_seed value from which HW address will be generated. + /// @param client_id_seed value from which client identifier will be + /// generated. + /// @param secs value to be stored in the "secs" field of the DHCPv4 message. + /// + /// @return Pointer to the created message. + dhcp::Pkt4Ptr createMessage4(const uint8_t msg_type, + const uint8_t hw_address_seed, + const uint8_t client_id_seed, + const uint16_t secs) const; + + /// @brief Creates test DHCPv4 query instance. + /// + /// @param hw_address_text HW address to be included in the query. It is + /// used in load balancing. + /// + /// @return Pointer to the DHCPv4 query instance. + dhcp::Pkt4Ptr createQuery4(const std::string& hw_address_text) const; + + /// @brief Creates test DHCPv4 query instance. + /// + /// @param hw_address HW address to be included in the query. It is used + /// in load balancing. + /// @param client_id optional client identifier. + dhcp::Pkt4Ptr createQuery4(const std::vector& hw_address, + const std::vector& client_id = + std::vector()) const; + + /// @brief Creates test DHCPv6 query instance. + /// + /// @param duid DUI to be included in the query. It is used in load balancing. + dhcp::Pkt6Ptr createQuery6(const std::vector& duid) const; + + /// @brief Generates simple DHCPv6 message. + /// + /// @param msg_type DHCPv6 message type to be created. + /// @param duid value from which DUID will be generated. + /// @param elapsed_time value of the Elapsed Time option. + /// + /// @return Pointer to the created message. + dhcp::Pkt6Ptr createMessage6(const uint8_t msg_type, + const uint8_t duid_seed, + const uint16_t elapsed_time) const; + + /// @brief Sets the DHCP multi-threading configuration in staging SrvConfig. + /// + /// @param enable_multi_threading value that maps to enable-multi-threading. + /// @param thread_pool_size value that maps to thread-pool-size. + /// @param queue_size value that maps to queue-size. + void setDHCPMultiThreadingConfig(bool enable_multi_threading, + uint32_t thread_pool_size = 0, + uint32_t queue_size = 16); + + /// @brief Replace a pattern in a configuration. + /// + /// @param config Configuration to patch. + /// @param from String to replace. + /// @param repl String which replaces all occurrences of from. + /// @result A copy of config where all occurrences of from were replaced + /// by repl. + std::string replaceInConfig(const std::string& config, + const std::string& from, + const std::string& repl) { + std::string result(config); + if (from.empty()) { + return (result); + } + for (;;) { + size_t where = result.find(from); + if (where == std::string::npos) { + return (result); + } + result.replace(where, from.size(), repl); + } + return (result); + } + + /// @brief Constructs JSON string for HA "multi-threading" element. + /// + /// Constructs a JSON string with the following content: + /// + /// ``` + /// "multi-threading" { + /// "enable-multi-threading": , + /// "dedicated-http-listener": , + /// "http-listener-threads": , + /// "http-client-threads": + /// }" + /// ``` + /// + /// @param enable_multi_threading value for enable-multi-threading. + /// @param http_dedicated_listener value for dedicated-http-listener. + /// @param http_listener_threads value for http-listener-threads + /// @param http_client_threads value for http-client-threads + /// + /// @return JSON string + std::string makeHAMtJson(bool enable_multi_threading, + bool http_dedicated_listener, + uint32_t http_listener_threads, + uint32_t http_client_threads); + + /// @brief Creates test DHCPv6 query instance. + /// + /// @param duid_text DUID to be included in the query. It is used in load + /// balancing. + /// + /// @return Pointer to the DHCPv6 query instance. + dhcp::Pkt6Ptr createQuery6(const std::string& duid_text) const; + + /// @brief Pointer to the IO service used in the tests. + asiolink::IOServicePtr io_service_; + + /// @brief Object holding a state of the DHCP service. + dhcp::NetworkStatePtr network_state_; +}; + +} // end of namespace isc::ha::test +} // end of namespace isc::ha +} // end of namespace isc diff --git a/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc b/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc new file mode 100644 index 0000000..a1cbe62 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc @@ -0,0 +1,91 @@ +// Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include + +#include +#include +#include + +using namespace isc::asiolink; +using namespace isc::dhcp; +using namespace isc::ha; + +namespace { + +// This test verifies that lease updates can be pushed to the queue, +// retrieved from the queue. +TEST(LeaseUpdateBacklogTest, pushAndPop) { + // Create the queue with limit of 5 lease updates. + LeaseUpdateBacklog backlog(5); + + // Add 5 lease updates. + for (auto i = 0; i < 5; ++i) { + IOAddress address(i + 1); + HWAddrPtr hwaddr = boost::make_shared(std::vector(6, static_cast(i)), + HTYPE_ETHER); + Lease4Ptr lease = boost::make_shared(address, hwaddr, ClientIdPtr(), 60, 0, 1); + // Some lease updates have type "Add", some have type "Delete". + ASSERT_TRUE(backlog.push(i % 2 ? LeaseUpdateBacklog::ADD : LeaseUpdateBacklog::DELETE, lease)); + EXPECT_FALSE(backlog.wasOverflown()); + } + + // Add 6th lease update. This should exceed the size limit. + IOAddress address("192.0.2.0"); + HWAddrPtr hwaddr = boost::make_shared(std::vector(6, static_cast(0xA)), + HTYPE_ETHER); + Lease4Ptr lease = boost::make_shared(address, hwaddr, ClientIdPtr(), 60, 0, 1); + ASSERT_FALSE(backlog.push(LeaseUpdateBacklog::ADD, lease)); + EXPECT_TRUE(backlog.wasOverflown()); + + // Try to pop all lease updates. + LeaseUpdateBacklog::OpType op_type; + for (auto i = 0; i < 5; ++i) { + auto lease = backlog.pop(op_type); + ASSERT_TRUE(lease); + ASSERT_EQ(i % 2 ? LeaseUpdateBacklog::ADD : LeaseUpdateBacklog::DELETE, op_type); + } + + // When trying to pop from an empty queue it should return null pointer. + lease = boost::dynamic_pointer_cast(backlog.pop(op_type)); + EXPECT_FALSE(lease); + EXPECT_TRUE(backlog.wasOverflown()); + + // Reset queue state. + ASSERT_NO_THROW(backlog.clear()); + EXPECT_FALSE(backlog.wasOverflown()); +} + +// This test verifies that all lease updates can be removed. +TEST(LeaseUpdateBacklogTest, clear) { + // Create the queue with limit of 5 lease updates. + LeaseUpdateBacklog backlog(5); + + // Add 5 lease updates. + for (auto i = 0; i < 3; ++i) { + IOAddress address(i + 1); + HWAddrPtr hwaddr = boost::make_shared(std::vector(6, static_cast(i)), + HTYPE_ETHER); + Lease4Ptr lease = boost::make_shared(address, hwaddr, ClientIdPtr(), 60, 0, 1); + ASSERT_TRUE(backlog.push(LeaseUpdateBacklog::ADD, lease)); + } + + // Make sure all lease updates have been added. + EXPECT_EQ(3, backlog.size()); + + // Remove lease updates. + ASSERT_NO_THROW(backlog.clear()); + + // There should be no lease updates. + EXPECT_EQ(0, backlog.size()); +} + +} // end of anonymous namespace diff --git a/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc b/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc new file mode 100644 index 0000000..4f7f2c4 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc @@ -0,0 +1,1029 @@ +// Copyright (C) 2018-2022 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace isc; +using namespace isc::data; +using namespace isc::dhcp; +using namespace isc::ha; +using namespace isc::ha::test; +using namespace util; + +namespace { + +/// @brief Test fixture class for @c QueryFilter class. +class QueryFilterTest : public HATest { +public: + /// @brief Constructor. + QueryFilterTest() { + MultiThreadingMgr::instance().setMode(false); + } + + /// @brief Destructor. + ~QueryFilterTest() { + MultiThreadingMgr::instance().setMode(false); + } + + /// @brief This test verifies that client identifier is used for load + /// balancing. + void loadBalancingClientIdThisPrimary(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is primary. + void loadBalancingThisPrimary(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is secondary. + void loadBalancingThisSecondary(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is backup. + /// @todo Expand these tests once we implement the actual load balancing to + /// verify which packets are in scope. + void loadBalancingThisBackup(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is primary. + void hotStandbyThisPrimary(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is standby. + void hotStandbyThisSecondary(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is backup. + void hotStandbyThisBackup(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is primary. + void loadBalancingThisPrimary6(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is secondary. + void loadBalancingThisSecondary6(); + + /// @brief This test verifies the case when load balancing is enabled and + /// this server is backup. + /// @todo Expand these tests once we implement the actual load balancing to + /// verify which packets are in scope. + void loadBalancingThisBackup6(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is primary. + void hotStandbyThisPrimary6(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is standby. + void hotStandbyThisSecondary6(); + + /// @brief This test verifies the case when hot-standby is enabled and this + /// server is backup. + void hotStandbyThisBackup6(); + + /// @brief This test verifies that it is possible to explicitly enable and + /// disable certain scopes. + void explicitlyServeScopes(); + + /// @brief This test verifies that load balancing only affects the scope of + /// DHCPv4 message types that HA cares about. + void loadBalancingHaTypes4(); + + /// @brief This test verifies that load balancing only affects the scope of + /// DHCPv6 message types that HA cares about. + void loadBalancingHaTypes6(); +}; + +void +QueryFilterTest::loadBalancingThisPrimary() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // By default the server1 should serve its own scope only. The + // server2 should serve its scope. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // If the query is in scope, increase the counter of packets in scope. + if (filter.inScope(query4, scope_class)) { + ASSERT_EQ("HA_server1", scope_class); + ASSERT_NE(scope_class, "HA_server2"); + ++in_scope; + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); + + // Simulate failover scenario. + filter.serveFailoverScopes(); + + // In the failover case, the server1 should also take responsibility for + // the server2's queries. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // Every single query mist be in scope. + ASSERT_TRUE(filter.inScope(query4, scope_class)); + } + + // However, the one that lacks HW address and client id should be out of + // scope. + Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 1234)); + EXPECT_FALSE(filter.inScope(query4, scope_class)); +} + +void +QueryFilterTest::loadBalancingClientIdThisPrimary() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // By default the server1 should serve its own scope only. The + // server2 should serve its scope. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Fixed HW address used in tests. + std::vector hw_address(HWAddr::ETHERNET_HWADDR_LEN); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random client identifier. + Pkt4Ptr query4 = createQuery4(hw_address, randomKey(8)); + // If the query is in scope, increase the counter of packets in scope. + if (filter.inScope(query4, scope_class)) { + ASSERT_EQ("HA_server1", scope_class); + ASSERT_NE(scope_class, "HA_server2"); + ++in_scope; + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); + + // Simulate failover scenario. + filter.serveFailoverScopes(); + + // In the failover case, the server1 should also take responsibility for + // the server2's queries. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random client identifier. + Pkt4Ptr query4 = createQuery4(hw_address, randomKey(8)); + // Every single query mist be in scope. + ASSERT_TRUE(filter.inScope(query4, scope_class)); + } +} + +void +QueryFilterTest::loadBalancingThisSecondary() { + HAConfigPtr config = createValidConfiguration(); + + // We're now a secondary server. + config->setThisServerName("server2"); + + QueryFilter filter(config); + + // By default the server2 should serve its own scope only. The + // server1 should serve its scope. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // If the query is in scope, increase the counter of packets in scope. + if (filter.inScope(query4, scope_class)) { + ASSERT_EQ("HA_server2", scope_class); + ASSERT_NE(scope_class, "HA_server1"); + ++in_scope; + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); + + // Simulate failover scenario. + filter.serveFailoverScopes(); + + // In this scenario, the server1 died, so the server2 should now serve + // both scopes. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // Every single query must be in scope. + ASSERT_TRUE(filter.inScope(query4, scope_class)); + } +} + +void +QueryFilterTest::loadBalancingThisBackup() { + HAConfigPtr config = createValidConfiguration(); + + config->setThisServerName("server3"); + + QueryFilter filter(config); + + // The backup server doesn't handle any DHCP traffic by default. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // None of the packets should be handlded by the backup server. + ASSERT_FALSE(filter.inScope(query4, scope_class)); + } + + // Simulate failover. Although, backup server never starts handling + // other server's traffic automatically, it can be manually instructed + // to do so. This simulates such scenario. + filter.serveFailoverScopes(); + + // The backup server now handles traffic of server 1 and server 2. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // Every single query must be in scope. + ASSERT_TRUE(filter.inScope(query4, scope_class)); + } +} + +void +QueryFilterTest::hotStandbyThisPrimary() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + + QueryFilter filter(config); + + Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66"); + + // By default, only the primary server is active. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + // It should process its queries. + EXPECT_TRUE(filter.inScope(query4, scope_class)); + + // Simulate failover scenario, in which the active server detects a + // failure of the standby server. This doesn't change anything in how + // the traffic is distributed. + filter.serveFailoverScopes(); + + // The server1 continues to process its own traffic. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query4, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); +} + +void +QueryFilterTest::hotStandbyThisSecondary() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + config->setThisServerName("server2"); + + QueryFilter filter(config); + + Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66"); + + // The server2 doesn't process any queries by default. The whole + // traffic is processed by the server1. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + EXPECT_FALSE(filter.inScope(query4, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); + + // Simulate failover case whereby the standby server detects a + // failure of the active server. + filter.serveFailoverScopes(); + + // The server2 now handles the traffic normally handled by the + // server1. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query4, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); +} + +void +QueryFilterTest::hotStandbyThisBackup() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + config->setThisServerName("server3"); + + QueryFilter filter(config); + + Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66"); + + // By default the backup server doesn't process any traffic. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + EXPECT_FALSE(filter.inScope(query4, scope_class)); + + // Simulate failover. Although, backup server never starts handling + // other server's traffic automatically, it can be manually instructed + // to do so. This simulates such scenario. + filter.serveFailoverScopes(); + + // The backup server now handles the entire traffic, i.e. the traffic + // that the primary server handles. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query4, scope_class)); +} + +void +QueryFilterTest::loadBalancingThisPrimary6() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // By default the server1 should serve its own scope only. The + // server2 should serve its scope. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random DUID. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + // If the query is in scope, increase the counter of packets in scope. + if (filter.inScope(query6, scope_class)) { + ASSERT_EQ("HA_server1", scope_class); + ASSERT_NE(scope_class, "HA_server2"); + ++in_scope; + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); + + // Simulate failover scenario. + filter.serveFailoverScopes(); + + // In the failover case, the server1 should also take responsibility for + // the server2's queries. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + // Every single query mist be in scope. + ASSERT_TRUE(filter.inScope(query6, scope_class)); + } + + // However, the one that lacks DUID should be out of scope. + Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 1234)); + EXPECT_FALSE(filter.inScope(query6, scope_class)); +} + +void +QueryFilterTest::loadBalancingThisSecondary6() { + HAConfigPtr config = createValidConfiguration(); + + // We're now a secondary server. + config->setThisServerName("server2"); + + QueryFilter filter(config); + + // By default the server2 should serve its own scope only. The + // server1 should serve its scope. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Count number of in scope packets. + unsigned in_scope = 0; + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + // If the query is in scope, increase the counter of packets in scope. + if (filter.inScope(query6, scope_class)) { + ASSERT_EQ("HA_server2", scope_class); + ASSERT_NE(scope_class, "HA_server1"); + ++in_scope; + } + } + + // We should have roughly 50/50 split of in scope and out of scope queries. + // However, we don't know exactly how many. To be safe we simply assume that + // we got more than 25% of in scope and more than 25% out of scope queries. + EXPECT_GT(in_scope, static_cast(queries_num / 4)); + EXPECT_GT(queries_num - in_scope, static_cast(queries_num / 4)); + + // Simulate failover scenario. + filter.serveFailoverScopes(); + + // In this scenario, the server1 died, so the server2 should now serve + // both scopes. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt6Ptr query6 = createQuery6(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // Every single query must be in scope. + ASSERT_TRUE(filter.inScope(query6, scope_class)); + } +} + +void +QueryFilterTest::loadBalancingThisBackup6() { + HAConfigPtr config = createValidConfiguration(); + + config->setThisServerName("server3"); + + QueryFilter filter(config); + + // The backup server doesn't handle any DHCP traffic by default. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Set the test size - 65535 queries. + const unsigned queries_num = 65535; + std::string scope_class; + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + // None of the packets should be handlded by the backup server. + ASSERT_FALSE(filter.inScope(query6, scope_class)); + } + + // Simulate failover. Although, backup server never starts handling + // other server's traffic automatically, it can be manually instructed + // to do so. This simulates such scenario. + filter.serveFailoverScopes(); + + // The backup server now handles traffic of server 1 and server 2. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Repeat the test, but this time all should be in scope. + for (unsigned i = 0; i < queries_num; ++i) { + // Create query with random HW address. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + // Every single query must be in scope. + ASSERT_TRUE(filter.inScope(query6, scope_class)); + } +} + +void +QueryFilterTest::hotStandbyThisPrimary6() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + + QueryFilter filter(config); + + Pkt6Ptr query6 = createQuery6("01:02:11:22:33:44:55:66"); + + // By default, only the primary server is active. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + // It should process its queries. + EXPECT_TRUE(filter.inScope(query6, scope_class)); + + // Simulate failover scenario, in which the active server detects a + // failure of the standby server. This doesn't change anything in how + // the traffic is distributed. + filter.serveFailoverScopes(); + + // The server1 continues to process its own traffic. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query6, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); +} + +void +QueryFilterTest::hotStandbyThisSecondary6() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + config->setThisServerName("server2"); + + QueryFilter filter(config); + + Pkt6Ptr query6 = createQuery6("01:02:11:22:33:44:55:66"); + + // The server2 doesn't process any queries by default. The whole + // traffic is processed by the server1. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + EXPECT_FALSE(filter.inScope(query6, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); + + // Simulate failover case whereby the standby server detects a + // failure of the active server. + filter.serveFailoverScopes(); + + // The server2 now handles the traffic normally handled by the + // server1. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query6, scope_class)); + EXPECT_EQ("HA_server1", scope_class); + EXPECT_NE(scope_class, "HA_server2"); +} + +void +QueryFilterTest::hotStandbyThisBackup6() { + HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY); + config->getPeerConfig("server2")->setRole("standby"); + config->setThisServerName("server3"); + + QueryFilter filter(config); + + Pkt6Ptr query6 = createQuery6(randomKey(10)); + + // By default the backup server doesn't process any traffic. + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + std::string scope_class; + + EXPECT_FALSE(filter.inScope(query6, scope_class)); + + // Simulate failover. Although, backup server never starts handling + // other server's traffic automatically, it can be manually instructed + // to do so. This simulates such scenario. + filter.serveFailoverScopes(); + + // The backup server now handles the entire traffic, i.e. the traffic + // that the primary server handles. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + EXPECT_TRUE(filter.inScope(query6, scope_class)); +} + +void +QueryFilterTest::explicitlyServeScopes() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // Initially, the scopes should be set according to the load + // balancing configuration. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Enable "server2" scope. + ASSERT_NO_THROW(filter.serveScope("server2")); + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Enable only "server2" scope. + ASSERT_NO_THROW(filter.serveScopeOnly("server2")); + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_TRUE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Explicitly enable selected scopes. + ASSERT_NO_THROW(filter.serveScopes({ "server1", "server3" })); + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_TRUE(filter.amServingScope("server3")); + + // Revert to defaults. + ASSERT_NO_THROW(filter.serveDefaultScopes()); + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Disable all scopes. + ASSERT_NO_THROW(filter.serveNoScopes()); + EXPECT_FALSE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Test negative cases. + EXPECT_THROW(filter.serveScope("unsupported"), BadValue); + EXPECT_THROW(filter.serveScopeOnly("unsupported"), BadValue); + EXPECT_THROW(filter.serveScopes({ "server1", "unsupported" }), BadValue); +} + +void +QueryFilterTest::loadBalancingHaTypes4() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // By default the server1 should serve its own scope only. The + // server2 should serve its scope. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Use DHCPDISCOVER to find MAC addresses in scope for server1 and server2. + Pkt4Ptr server1_pkt; + Pkt4Ptr server2_pkt; + const unsigned max_scope_tries = 100; + for (unsigned i = 0; i < max_scope_tries; ++i) { + // Create query with random HW address. + std::string scope_class; + Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN)); + // If the query is in scope then we're done. + if (filter.inScope(query4, scope_class)) { + ASSERT_EQ("HA_server1", scope_class); + server1_pkt = query4; + if (server2_pkt) { + break; + } + } else { + ASSERT_EQ("HA_server2", scope_class); + server2_pkt = query4; + if (server1_pkt) { + break; + } + } + } + + ASSERT_TRUE(server1_pkt && server2_pkt) << "do not have both scopes in " + << max_scope_tries << ", load balance broken?"; + + // Iterate over message types. While setting message type to zero is + // semantically invalid, it is useful for testing here. Similarly we exceed + // DHCP_TYPES_EOF just to be sure. + for (uint8_t msg_type = 0; msg_type < DHCP_TYPES_EOF + 2; ++msg_type) { + // All message types should be in scope for server1. + server1_pkt->setType(msg_type); + + std::string scope_class; + bool is_in_scope = filter.inScope(server1_pkt, scope_class); + ASSERT_EQ("HA_server1", scope_class); + EXPECT_TRUE(is_in_scope); + + server2_pkt->setType(msg_type); + scope_class.clear(); + is_in_scope = filter.inScope(server2_pkt, scope_class); + switch (msg_type) { + case DHCPDISCOVER: + case DHCPREQUEST: + case DHCPDECLINE: + case DHCPRELEASE: + case DHCPINFORM: + // HA message types should be in scope for server2. + ASSERT_EQ("HA_server2", scope_class); + EXPECT_FALSE(is_in_scope); + break; + default: + // Non HA message types should be in scope for server1. + ASSERT_EQ("HA_server1", scope_class); + EXPECT_TRUE(is_in_scope); + break; + } + } +} + +void +QueryFilterTest::loadBalancingHaTypes6() { + HAConfigPtr config = createValidConfiguration(); + + QueryFilter filter(config); + + // By default the server1 should serve its own scope only. The + // server2 should serve its scope. + EXPECT_TRUE(filter.amServingScope("server1")); + EXPECT_FALSE(filter.amServingScope("server2")); + EXPECT_FALSE(filter.amServingScope("server3")); + + // Use DHCPV6_SOLICIT to find MAC addresses in scope for server1 and server2. + Pkt6Ptr server1_pkt; + Pkt6Ptr server2_pkt; + const unsigned max_scope_tries = 100; + for (unsigned i = 0; i < max_scope_tries; ++i) { + // Create query with random HW address. + std::string scope_class; + + // Create query with random DUID. + Pkt6Ptr query6 = createQuery6(randomKey(10)); + if (filter.inScope(query6, scope_class)) { + ASSERT_EQ("HA_server1", scope_class); + // In scope for server1, save it. + server1_pkt = query6; + if (server2_pkt) { + // Have both, we're done. + break; + } + } else { + ASSERT_EQ("HA_server2", scope_class); + // In scope for server2, save it. + server2_pkt = query6; + if (server1_pkt) { + // Have both, we're done. + break; + } + } + } + + ASSERT_TRUE(server1_pkt && server2_pkt) << "do not have both scopes in " + << max_scope_tries << ", load balance broken?"; + + // Iterate over message types. While setting message type to zero is + // semantically invalid, it is useful for testing here. Similarly we exceed + // DHCPV6_TYPES_EOF just to be sure. + for (uint8_t msg_type = 0; msg_type < DHCPV6_TYPES_EOF + 2; ++msg_type) { + // All message types should be in scope for server1. + server1_pkt->setType(msg_type); + + std::string scope_class; + bool is_in_scope = filter.inScope(server1_pkt, scope_class); + ASSERT_EQ("HA_server1", scope_class); + EXPECT_TRUE(is_in_scope); + + server2_pkt->setType(msg_type); + scope_class.clear(); + is_in_scope = filter.inScope(server2_pkt, scope_class); + switch (msg_type) { + case DHCPV6_SOLICIT: + case DHCPV6_REQUEST: + case DHCPV6_CONFIRM: + case DHCPV6_RENEW: + case DHCPV6_REBIND: + case DHCPV6_RELEASE: + case DHCPV6_DECLINE: + // HA message types should be in scope for server2. + ASSERT_EQ("HA_server2", scope_class); + EXPECT_FALSE(is_in_scope); + break; + default: + // Non HA message types should be in scope for server1. + ASSERT_EQ("HA_server1", scope_class); + EXPECT_TRUE(is_in_scope); + break; + } + } +} + +TEST_F(QueryFilterTest, loadBalancingClientIdThisPrimary) { + loadBalancingClientIdThisPrimary(); +} + +TEST_F(QueryFilterTest, loadBalancingClientIdThisPrimaryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingClientIdThisPrimary(); +} + +TEST_F(QueryFilterTest, loadBalancingThisPrimary) { + loadBalancingThisPrimary(); +} + +TEST_F(QueryFilterTest, loadBalancingThisPrimaryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisPrimary(); +} + +TEST_F(QueryFilterTest, loadBalancingThisSecondary) { + loadBalancingThisSecondary(); +} + +TEST_F(QueryFilterTest, loadBalancingThisSecondaryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisSecondary(); +} + +TEST_F(QueryFilterTest, loadBalancingThisBackup) { + loadBalancingThisBackup(); +} + +TEST_F(QueryFilterTest, loadBalancingThisBackupMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisBackup(); +} + +TEST_F(QueryFilterTest, hotStandbyThisPrimary) { + hotStandbyThisPrimary(); +} + +TEST_F(QueryFilterTest, hotStandbyThisPrimaryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisPrimary(); +} + +TEST_F(QueryFilterTest, hotStandbyThisSecondary) { + hotStandbyThisSecondary(); +} + +TEST_F(QueryFilterTest, hotStandbyThisSecondaryMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisSecondary(); +} + +TEST_F(QueryFilterTest, hotStandbyThisBackup) { + hotStandbyThisBackup(); +} + +TEST_F(QueryFilterTest, hotStandbyThisBackupMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisBackup(); +} + +TEST_F(QueryFilterTest, loadBalancingThisPrimary6) { + loadBalancingThisPrimary6(); +} + +TEST_F(QueryFilterTest, loadBalancingThisPrimary6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisPrimary6(); +} + +TEST_F(QueryFilterTest, loadBalancingThisSecondary6) { + loadBalancingThisSecondary6(); +} + +TEST_F(QueryFilterTest, loadBalancingThisSecondary6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisSecondary6(); +} + +TEST_F(QueryFilterTest, loadBalancingThisBackup6) { + loadBalancingThisBackup6(); +} + +TEST_F(QueryFilterTest, loadBalancingThisBackup6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingThisBackup6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisPrimary6) { + hotStandbyThisPrimary6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisPrimary6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisPrimary6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisSecondary6) { + hotStandbyThisSecondary6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisSecondary6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisSecondary6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisBackup6) { + hotStandbyThisBackup6(); +} + +TEST_F(QueryFilterTest, hotStandbyThisBackup6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + hotStandbyThisBackup6(); +} + +TEST_F(QueryFilterTest, explicitlyServeScopes) { + explicitlyServeScopes(); +} + +TEST_F(QueryFilterTest, explicitlyServeScopesMultiThreading) { + MultiThreadingMgr::instance().setMode(true); + explicitlyServeScopes(); +} + +TEST_F(QueryFilterTest, loadBalancingHaTypes4) { + loadBalancingHaTypes4(); +} + +TEST_F(QueryFilterTest, loadBalancingHaTypes4MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingHaTypes4(); +} + +TEST_F(QueryFilterTest, loadBalancingHaTypes6) { + loadBalancingHaTypes6(); +} + +TEST_F(QueryFilterTest, loadBalancingHaTypes6MultiThreading) { + MultiThreadingMgr::instance().setMode(true); + loadBalancingHaTypes6(); +} + +} diff --git a/src/hooks/dhcp/high_availability/tests/run_unittests.cc b/src/hooks/dhcp/high_availability/tests/run_unittests.cc new file mode 100644 index 0000000..3b1baf9 --- /dev/null +++ b/src/hooks/dhcp/high_availability/tests/run_unittests.cc @@ -0,0 +1,19 @@ +// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include + +int +main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + isc::log::initLogger(); + int result = RUN_ALL_TESTS(); + + return (result); +} -- cgit v1.2.3