summaryrefslogtreecommitdiffstats
path: root/src/hooks/dhcp/high_availability/tests
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 12:15:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 12:15:43 +0000
commitf5f56e1a1c4d9e9496fcb9d81131066a964ccd23 (patch)
tree49e44c6f87febed37efb953ab5485aa49f6481a7 /src/hooks/dhcp/high_availability/tests
parentInitial commit. (diff)
downloadisc-kea-f5f56e1a1c4d9e9496fcb9d81131066a964ccd23.tar.xz
isc-kea-f5f56e1a1c4d9e9496fcb9d81131066a964ccd23.zip
Adding upstream version 2.4.1.upstream/2.4.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/hooks/dhcp/high_availability/tests')
-rw-r--r--src/hooks/dhcp/high_availability/tests/Makefile.am70
-rw-r--r--src/hooks/dhcp/high_availability/tests/Makefile.in1217
-rw-r--r--src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc530
-rw-r--r--src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc1201
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc1879
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc814
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc561
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc8037
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_test.cc378
-rw-r--r--src/hooks/dhcp/high_availability/tests/ha_test.h304
-rw-r--r--src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc91
-rw-r--r--src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc1029
-rw-r--r--src/hooks/dhcp/high_availability/tests/run_unittests.cc19
13 files changed, 16130 insertions, 0 deletions
diff --git a/src/hooks/dhcp/high_availability/tests/Makefile.am b/src/hooks/dhcp/high_availability/tests/Makefile.am
new file mode 100644
index 0000000..f99f8eb
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/Makefile.am
@@ -0,0 +1,70 @@
+SUBDIRS = .
+
+AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib
+AM_CPPFLAGS += -I$(top_builddir)/src/hooks/dhcp/high_availability -I$(top_srcdir)/src/hooks/dhcp/high_availability
+AM_CPPFLAGS += $(BOOST_INCLUDES) $(CRYPTO_CFLAGS) $(CRYPTO_INCLUDES)
+AM_CPPFLAGS += -DLIBDHCP_HA_SO=\"$(abs_top_builddir)/src/hooks/dhcp/high_availability/.libs/libdhcp_ha.so\"
+AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
+TEST_CA_DIR = $(abs_top_srcdir)/src/lib/asiolink/testutils/ca
+AM_CPPFLAGS += -DTEST_CA_DIR=\"$(TEST_CA_DIR)\"
+TEST_HTTP_DIR = $(abs_top_srcdir)/src/lib/http/tests/testdata
+AM_CPPFLAGS += -DTEST_HTTP_DIR=\"$(TEST_HTTP_DIR)\"
+
+AM_CXXFLAGS = $(KEA_CXXFLAGS)
+
+if USE_STATIC_LINK
+AM_LDFLAGS = -static
+endif
+
+# Unit test data files need to get installed.
+EXTRA_DIST =
+
+CLEANFILES = *.gcno *.gcda
+
+TESTS_ENVIRONMENT = $(LIBTOOL) --mode=execute $(VALGRIND_COMMAND)
+
+TESTS =
+if HAVE_GTEST
+TESTS += ha_unittests
+
+ha_unittests_SOURCES = command_creator_unittest.cc
+ha_unittests_SOURCES += communication_state_unittest.cc
+ha_unittests_SOURCES += ha_config_unittest.cc
+ha_unittests_SOURCES += ha_impl_unittest.cc
+ha_unittests_SOURCES += ha_service_unittest.cc
+ha_unittests_SOURCES += ha_test.cc ha_test.h
+ha_unittests_SOURCES += ha_mt_unittest.cc
+ha_unittests_SOURCES += lease_update_backlog_unittest.cc
+ha_unittests_SOURCES += query_filter_unittest.cc
+ha_unittests_SOURCES += run_unittests.cc
+
+ha_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES) $(LOG4CPLUS_INCLUDES)
+
+ha_unittests_LDFLAGS = $(AM_LDFLAGS) $(CRYPTO_LDFLAGS) $(GTEST_LDFLAGS)
+
+ha_unittests_CXXFLAGS = $(AM_CXXFLAGS)
+
+ha_unittests_LDADD = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/process/libkea-process.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/eval/libkea-eval.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/stats/libkea-stats.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/config/libkea-cfgclient.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/http/libkea-http.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/hooks/libkea-hooks.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/database/libkea-database.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/cc/libkea-cc.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/asiolink/libkea-asiolink.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/dns/libkea-dns++.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/log/libkea-log.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/util/libkea-util.la
+ha_unittests_LDADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
+ha_unittests_LDADD += $(LOG4CPLUS_LIBS)
+ha_unittests_LDADD += $(CRYPTO_LIBS)
+ha_unittests_LDADD += $(BOOST_LIBS)
+ha_unittests_LDADD += $(GTEST_LDADD)
+endif
+noinst_PROGRAMS = $(TESTS)
diff --git a/src/hooks/dhcp/high_availability/tests/Makefile.in b/src/hooks/dhcp/high_availability/tests/Makefile.in
new file mode 100644
index 0000000..a346891
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/Makefile.in
@@ -0,0 +1,1217 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+ if test -z '$(MAKELEVEL)'; then \
+ false; \
+ elif test -n '$(MAKE_HOST)'; then \
+ true; \
+ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+ true; \
+ else \
+ false; \
+ fi; \
+}
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+TESTS = $(am__EXEEXT_1)
+@HAVE_GTEST_TRUE@am__append_1 = ha_unittests
+noinst_PROGRAMS = $(am__EXEEXT_2)
+subdir = src/hooks/dhcp/high_availability/tests
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4macros/ax_boost_for_kea.m4 \
+ $(top_srcdir)/m4macros/ax_cpp11.m4 \
+ $(top_srcdir)/m4macros/ax_cpp20.m4 \
+ $(top_srcdir)/m4macros/ax_crypto.m4 \
+ $(top_srcdir)/m4macros/ax_find_library.m4 \
+ $(top_srcdir)/m4macros/ax_gssapi.m4 \
+ $(top_srcdir)/m4macros/ax_gtest.m4 \
+ $(top_srcdir)/m4macros/ax_isc_rpath.m4 \
+ $(top_srcdir)/m4macros/ax_netconf.m4 \
+ $(top_srcdir)/m4macros/libtool.m4 \
+ $(top_srcdir)/m4macros/ltoptions.m4 \
+ $(top_srcdir)/m4macros/ltsugar.m4 \
+ $(top_srcdir)/m4macros/ltversion.m4 \
+ $(top_srcdir)/m4macros/lt~obsolete.m4 \
+ $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+@HAVE_GTEST_TRUE@am__EXEEXT_1 = ha_unittests$(EXEEXT)
+am__EXEEXT_2 = $(am__EXEEXT_1)
+PROGRAMS = $(noinst_PROGRAMS)
+am__ha_unittests_SOURCES_DIST = command_creator_unittest.cc \
+ communication_state_unittest.cc ha_config_unittest.cc \
+ ha_impl_unittest.cc ha_service_unittest.cc ha_test.cc \
+ ha_test.h ha_mt_unittest.cc lease_update_backlog_unittest.cc \
+ query_filter_unittest.cc run_unittests.cc
+@HAVE_GTEST_TRUE@am_ha_unittests_OBJECTS = ha_unittests-command_creator_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-communication_state_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-ha_config_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-ha_impl_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-ha_service_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-ha_test.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-ha_mt_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-lease_update_backlog_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-query_filter_unittest.$(OBJEXT) \
+@HAVE_GTEST_TRUE@ ha_unittests-run_unittests.$(OBJEXT)
+ha_unittests_OBJECTS = $(am_ha_unittests_OBJECTS)
+am__DEPENDENCIES_1 =
+@HAVE_GTEST_TRUE@ha_unittests_DEPENDENCIES = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/process/libkea-process.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/eval/libkea-eval.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/stats/libkea-stats.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/config/libkea-cfgclient.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/http/libkea-http.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/hooks/libkea-hooks.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/database/libkea-database.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cc/libkea-cc.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/asiolink/libkea-asiolink.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dns/libkea-dns++.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/log/libkea-log.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/util/libkea-util.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/exceptions/libkea-exceptions.la \
+@HAVE_GTEST_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
+@HAVE_GTEST_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 =
+ha_unittests_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(ha_unittests_CXXFLAGS) \
+ $(CXXFLAGS) $(ha_unittests_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = \
+ ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-ha_test.Po \
+ ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po \
+ ./$(DEPDIR)/ha_unittests-run_unittests.Po
+am__mv = mv -f
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CXXFLAGS) $(CXXFLAGS)
+AM_V_CXX = $(am__v_CXX_@AM_V@)
+am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@)
+am__v_CXX_0 = @echo " CXX " $@;
+am__v_CXX_1 =
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+ $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CXXLD = $(am__v_CXXLD_@AM_V@)
+am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@)
+am__v_CXXLD_0 = @echo " CXXLD " $@;
+am__v_CXXLD_1 =
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo " CC " $@;
+am__v_CC_1 =
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo " CCLD " $@;
+am__v_CCLD_1 =
+SOURCES = $(ha_unittests_SOURCES)
+DIST_SOURCES = $(am__ha_unittests_SOURCES_DIST)
+RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \
+ ctags-recursive dvi-recursive html-recursive info-recursive \
+ install-data-recursive install-dvi-recursive \
+ install-exec-recursive install-html-recursive \
+ install-info-recursive install-pdf-recursive \
+ install-ps-recursive install-recursive installcheck-recursive \
+ installdirs-recursive pdf-recursive ps-recursive \
+ tags-recursive uninstall-recursive
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \
+ distclean-recursive maintainer-clean-recursive
+am__recursive_targets = \
+ $(RECURSIVE_TARGETS) \
+ $(RECURSIVE_CLEAN_TARGETS) \
+ $(am__extra_recursive_targets)
+AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \
+ distdir distdir-am
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
+am__tty_colors = { \
+ $(am__tty_colors_dummy); \
+ if test "X$(AM_COLOR_TESTS)" = Xno; then \
+ am__color_tests=no; \
+ elif test "X$(AM_COLOR_TESTS)" = Xalways; then \
+ am__color_tests=yes; \
+ elif test "X$$TERM" != Xdumb && { test -t 1; } 2>/dev/null; then \
+ am__color_tests=yes; \
+ fi; \
+ if test $$am__color_tests = yes; then \
+ red=''; \
+ grn=''; \
+ lgn=''; \
+ blu=''; \
+ mgn=''; \
+ brg=''; \
+ std=''; \
+ fi; \
+}
+DIST_SUBDIRS = $(SUBDIRS)
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+am__relativize = \
+ dir0=`pwd`; \
+ sed_first='s,^\([^/]*\)/.*$$,\1,'; \
+ sed_rest='s,^[^/]*/*,,'; \
+ sed_last='s,^.*/\([^/]*\)$$,\1,'; \
+ sed_butlast='s,/*[^/]*$$,,'; \
+ while test -n "$$dir1"; do \
+ first=`echo "$$dir1" | sed -e "$$sed_first"`; \
+ if test "$$first" != "."; then \
+ if test "$$first" = ".."; then \
+ dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \
+ dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \
+ else \
+ first2=`echo "$$dir2" | sed -e "$$sed_first"`; \
+ if test "$$first2" = "$$first"; then \
+ dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \
+ else \
+ dir2="../$$dir2"; \
+ fi; \
+ dir0="$$dir0"/"$$first"; \
+ fi; \
+ fi; \
+ dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \
+ done; \
+ reldir="$$dir2"
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AR = @AR@
+ASCIIDOC = @ASCIIDOC@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BOOST_INCLUDES = @BOOST_INCLUDES@
+BOOST_LIBS = @BOOST_LIBS@
+BOTAN_TOOL = @BOTAN_TOOL@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CONTRIB_DIR = @CONTRIB_DIR@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPTO_CFLAGS = @CRYPTO_CFLAGS@
+CRYPTO_INCLUDES = @CRYPTO_INCLUDES@
+CRYPTO_LDFLAGS = @CRYPTO_LDFLAGS@
+CRYPTO_LIBS = @CRYPTO_LIBS@
+CRYPTO_PACKAGE = @CRYPTO_PACKAGE@
+CRYPTO_RPATH = @CRYPTO_RPATH@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DISTCHECK_BOOST_CONFIGURE_FLAG = @DISTCHECK_BOOST_CONFIGURE_FLAG@
+DISTCHECK_CONTRIB_CONFIGURE_FLAG = @DISTCHECK_CONTRIB_CONFIGURE_FLAG@
+DISTCHECK_CRYPTO_CONFIGURE_FLAG = @DISTCHECK_CRYPTO_CONFIGURE_FLAG@
+DISTCHECK_GSSAPI_CONFIGURE_FLAG = @DISTCHECK_GSSAPI_CONFIGURE_FLAG@
+DISTCHECK_GTEST_CONFIGURE_FLAG = @DISTCHECK_GTEST_CONFIGURE_FLAG@
+DISTCHECK_KEA_SHELL_CONFIGURE_FLAG = @DISTCHECK_KEA_SHELL_CONFIGURE_FLAG@
+DISTCHECK_LIBYANGCPP_CONFIGURE_FLAG = @DISTCHECK_LIBYANGCPP_CONFIGURE_FLAG@
+DISTCHECK_LIBYANG_CONFIGURE_FLAG = @DISTCHECK_LIBYANG_CONFIGURE_FLAG@
+DISTCHECK_LOG4CPLUS_CONFIGURE_FLAG = @DISTCHECK_LOG4CPLUS_CONFIGURE_FLAG@
+DISTCHECK_MYSQL_CONFIGURE_FLAG = @DISTCHECK_MYSQL_CONFIGURE_FLAG@
+DISTCHECK_PERFDHCP_CONFIGURE_FLAG = @DISTCHECK_PERFDHCP_CONFIGURE_FLAG@
+DISTCHECK_PGSQL_CONFIGURE_FLAG = @DISTCHECK_PGSQL_CONFIGURE_FLAG@
+DISTCHECK_PREMIUM_CONFIGURE_FLAG = @DISTCHECK_PREMIUM_CONFIGURE_FLAG@
+DISTCHECK_SYSREPOCPP_CONFIGURE_FLAG = @DISTCHECK_SYSREPOCPP_CONFIGURE_FLAG@
+DISTCHECK_SYSREPO_CONFIGURE_FLAG = @DISTCHECK_SYSREPO_CONFIGURE_FLAG@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+GENHTML = @GENHTML@
+GREP = @GREP@
+GSSAPI_CFLAGS = @GSSAPI_CFLAGS@
+GSSAPI_LIBS = @GSSAPI_LIBS@
+GTEST_CONFIG = @GTEST_CONFIG@
+GTEST_INCLUDES = @GTEST_INCLUDES@
+GTEST_LDADD = @GTEST_LDADD@
+GTEST_LDFLAGS = @GTEST_LDFLAGS@
+GTEST_SOURCE = @GTEST_SOURCE@
+HAVE_NETCONF = @HAVE_NETCONF@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KEA_CXXFLAGS = @KEA_CXXFLAGS@
+KEA_SRCID = @KEA_SRCID@
+KRB5_CONFIG = @KRB5_CONFIG@
+LCOV = @LCOV@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LEX = @LEX@
+LEXLIB = @LEXLIB@
+LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LIBYANGCPP_CPPFLAGS = @LIBYANGCPP_CPPFLAGS@
+LIBYANGCPP_INCLUDEDIR = @LIBYANGCPP_INCLUDEDIR@
+LIBYANGCPP_LIBS = @LIBYANGCPP_LIBS@
+LIBYANGCPP_PREFIX = @LIBYANGCPP_PREFIX@
+LIBYANGCPP_VERSION = @LIBYANGCPP_VERSION@
+LIBYANG_CPPFLAGS = @LIBYANG_CPPFLAGS@
+LIBYANG_INCLUDEDIR = @LIBYANG_INCLUDEDIR@
+LIBYANG_LIBS = @LIBYANG_LIBS@
+LIBYANG_PREFIX = @LIBYANG_PREFIX@
+LIBYANG_VERSION = @LIBYANG_VERSION@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LOG4CPLUS_INCLUDES = @LOG4CPLUS_INCLUDES@
+LOG4CPLUS_LIBS = @LOG4CPLUS_LIBS@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MYSQL_CPPFLAGS = @MYSQL_CPPFLAGS@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PACKAGE_VERSION_TYPE = @PACKAGE_VERSION_TYPE@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PDFLATEX = @PDFLATEX@
+PERL = @PERL@
+PGSQL_CPPFLAGS = @PGSQL_CPPFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PKGPYTHONDIR = @PKGPYTHONDIR@
+PKG_CONFIG = @PKG_CONFIG@
+PLANTUML = @PLANTUML@
+PREMIUM_DIR = @PREMIUM_DIR@
+PYTHON = @PYTHON@
+PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@
+PYTHON_PLATFORM = @PYTHON_PLATFORM@
+PYTHON_PREFIX = @PYTHON_PREFIX@
+PYTHON_VERSION = @PYTHON_VERSION@
+RANLIB = @RANLIB@
+SED = @SED@
+SEP = @SEP@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SPHINXBUILD = @SPHINXBUILD@
+SRPD_PLUGINS_PATH = @SRPD_PLUGINS_PATH@
+SR_PLUGINS_PATH = @SR_PLUGINS_PATH@
+SR_REPO_PATH = @SR_REPO_PATH@
+STRIP = @STRIP@
+SYSREPOCPP_CPPFLAGS = @SYSREPOCPP_CPPFLAGS@
+SYSREPOCPP_INCLUDEDIR = @SYSREPOCPP_INCLUDEDIR@
+SYSREPOCPP_LIBS = @SYSREPOCPP_LIBS@
+SYSREPOCPP_PREFIX = @SYSREPOCPP_PREFIX@
+SYSREPOCPP_VERSION = @SYSREPOCPP_VERSION@
+SYSREPO_CPPFLAGS = @SYSREPO_CPPFLAGS@
+SYSREPO_INCLUDEDIR = @SYSREPO_INCLUDEDIR@
+SYSREPO_LIBS = @SYSREPO_LIBS@
+SYSREPO_PREFIX = @SYSREPO_PREFIX@
+SYSREPO_VERSION = @SYSREPO_VERSION@
+USE_LCOV = @USE_LCOV@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+WARNING_GCC_44_STRICT_ALIASING_CFLAG = @WARNING_GCC_44_STRICT_ALIASING_CFLAG@
+YACC = @YACC@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pkgpyexecdir = @pkgpyexecdir@
+pkgpythondir = @pkgpythondir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pyexecdir = @pyexecdir@
+pythondir = @pythondir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+SUBDIRS = .
+AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib \
+ -I$(top_builddir)/src/hooks/dhcp/high_availability \
+ -I$(top_srcdir)/src/hooks/dhcp/high_availability \
+ $(BOOST_INCLUDES) $(CRYPTO_CFLAGS) $(CRYPTO_INCLUDES) \
+ -DLIBDHCP_HA_SO=\"$(abs_top_builddir)/src/hooks/dhcp/high_availability/.libs/libdhcp_ha.so\" \
+ -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\" \
+ -DTEST_CA_DIR=\"$(TEST_CA_DIR)\" \
+ -DTEST_HTTP_DIR=\"$(TEST_HTTP_DIR)\"
+TEST_CA_DIR = $(abs_top_srcdir)/src/lib/asiolink/testutils/ca
+TEST_HTTP_DIR = $(abs_top_srcdir)/src/lib/http/tests/testdata
+AM_CXXFLAGS = $(KEA_CXXFLAGS)
+@USE_STATIC_LINK_TRUE@AM_LDFLAGS = -static
+
+# Unit test data files need to get installed.
+EXTRA_DIST =
+CLEANFILES = *.gcno *.gcda
+TESTS_ENVIRONMENT = $(LIBTOOL) --mode=execute $(VALGRIND_COMMAND)
+@HAVE_GTEST_TRUE@ha_unittests_SOURCES = command_creator_unittest.cc \
+@HAVE_GTEST_TRUE@ communication_state_unittest.cc \
+@HAVE_GTEST_TRUE@ ha_config_unittest.cc ha_impl_unittest.cc \
+@HAVE_GTEST_TRUE@ ha_service_unittest.cc ha_test.cc ha_test.h \
+@HAVE_GTEST_TRUE@ ha_mt_unittest.cc \
+@HAVE_GTEST_TRUE@ lease_update_backlog_unittest.cc \
+@HAVE_GTEST_TRUE@ query_filter_unittest.cc run_unittests.cc
+@HAVE_GTEST_TRUE@ha_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES) $(LOG4CPLUS_INCLUDES)
+@HAVE_GTEST_TRUE@ha_unittests_LDFLAGS = $(AM_LDFLAGS) $(CRYPTO_LDFLAGS) $(GTEST_LDFLAGS)
+@HAVE_GTEST_TRUE@ha_unittests_CXXFLAGS = $(AM_CXXFLAGS)
+@HAVE_GTEST_TRUE@ha_unittests_LDADD = $(top_builddir)/src/hooks/dhcp/high_availability/libha.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/process/libkea-process.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/eval/libkea-eval.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp_ddns/libkea-dhcp_ddns.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/stats/libkea-stats.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/config/libkea-cfgclient.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/http/libkea-http.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/hooks/libkea-hooks.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/database/libkea-database.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cc/libkea-cc.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/asiolink/libkea-asiolink.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/dns/libkea-dns++.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/cryptolink/libkea-cryptolink.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/log/libkea-log.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/util/libkea-util.la \
+@HAVE_GTEST_TRUE@ $(top_builddir)/src/lib/exceptions/libkea-exceptions.la \
+@HAVE_GTEST_TRUE@ $(LOG4CPLUS_LIBS) $(CRYPTO_LIBS) \
+@HAVE_GTEST_TRUE@ $(BOOST_LIBS) $(GTEST_LDADD)
+all: all-recursive
+
+.SUFFIXES:
+.SUFFIXES: .cc .lo .o .obj
+$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/hooks/dhcp/high_availability/tests/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign src/hooks/dhcp/high_availability/tests/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+ @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+
+ha_unittests$(EXEEXT): $(ha_unittests_OBJECTS) $(ha_unittests_DEPENDENCIES) $(EXTRA_ha_unittests_DEPENDENCIES)
+ @rm -f ha_unittests$(EXEEXT)
+ $(AM_V_CXXLD)$(ha_unittests_LINK) $(ha_unittests_OBJECTS) $(ha_unittests_LDADD) $(LIBS)
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-command_creator_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-communication_state_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_config_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_service_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-ha_test.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-query_filter_unittest.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ha_unittests-run_unittests.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+ @$(MKDIR_P) $(@D)
+ @echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.cc.o:
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $<
+
+.cc.obj:
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cc.lo:
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $<
+
+ha_unittests-command_creator_unittest.o: command_creator_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-command_creator_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo -c -o ha_unittests-command_creator_unittest.o `test -f 'command_creator_unittest.cc' || echo '$(srcdir)/'`command_creator_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo $(DEPDIR)/ha_unittests-command_creator_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='command_creator_unittest.cc' object='ha_unittests-command_creator_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-command_creator_unittest.o `test -f 'command_creator_unittest.cc' || echo '$(srcdir)/'`command_creator_unittest.cc
+
+ha_unittests-command_creator_unittest.obj: command_creator_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-command_creator_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo -c -o ha_unittests-command_creator_unittest.obj `if test -f 'command_creator_unittest.cc'; then $(CYGPATH_W) 'command_creator_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/command_creator_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-command_creator_unittest.Tpo $(DEPDIR)/ha_unittests-command_creator_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='command_creator_unittest.cc' object='ha_unittests-command_creator_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-command_creator_unittest.obj `if test -f 'command_creator_unittest.cc'; then $(CYGPATH_W) 'command_creator_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/command_creator_unittest.cc'; fi`
+
+ha_unittests-communication_state_unittest.o: communication_state_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-communication_state_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo -c -o ha_unittests-communication_state_unittest.o `test -f 'communication_state_unittest.cc' || echo '$(srcdir)/'`communication_state_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo $(DEPDIR)/ha_unittests-communication_state_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='communication_state_unittest.cc' object='ha_unittests-communication_state_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-communication_state_unittest.o `test -f 'communication_state_unittest.cc' || echo '$(srcdir)/'`communication_state_unittest.cc
+
+ha_unittests-communication_state_unittest.obj: communication_state_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-communication_state_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo -c -o ha_unittests-communication_state_unittest.obj `if test -f 'communication_state_unittest.cc'; then $(CYGPATH_W) 'communication_state_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/communication_state_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-communication_state_unittest.Tpo $(DEPDIR)/ha_unittests-communication_state_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='communication_state_unittest.cc' object='ha_unittests-communication_state_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-communication_state_unittest.obj `if test -f 'communication_state_unittest.cc'; then $(CYGPATH_W) 'communication_state_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/communication_state_unittest.cc'; fi`
+
+ha_unittests-ha_config_unittest.o: ha_config_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_config_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo -c -o ha_unittests-ha_config_unittest.o `test -f 'ha_config_unittest.cc' || echo '$(srcdir)/'`ha_config_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo $(DEPDIR)/ha_unittests-ha_config_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_config_unittest.cc' object='ha_unittests-ha_config_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_config_unittest.o `test -f 'ha_config_unittest.cc' || echo '$(srcdir)/'`ha_config_unittest.cc
+
+ha_unittests-ha_config_unittest.obj: ha_config_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_config_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo -c -o ha_unittests-ha_config_unittest.obj `if test -f 'ha_config_unittest.cc'; then $(CYGPATH_W) 'ha_config_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_config_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_config_unittest.Tpo $(DEPDIR)/ha_unittests-ha_config_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_config_unittest.cc' object='ha_unittests-ha_config_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_config_unittest.obj `if test -f 'ha_config_unittest.cc'; then $(CYGPATH_W) 'ha_config_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_config_unittest.cc'; fi`
+
+ha_unittests-ha_impl_unittest.o: ha_impl_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_impl_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo -c -o ha_unittests-ha_impl_unittest.o `test -f 'ha_impl_unittest.cc' || echo '$(srcdir)/'`ha_impl_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo $(DEPDIR)/ha_unittests-ha_impl_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_impl_unittest.cc' object='ha_unittests-ha_impl_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_impl_unittest.o `test -f 'ha_impl_unittest.cc' || echo '$(srcdir)/'`ha_impl_unittest.cc
+
+ha_unittests-ha_impl_unittest.obj: ha_impl_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_impl_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo -c -o ha_unittests-ha_impl_unittest.obj `if test -f 'ha_impl_unittest.cc'; then $(CYGPATH_W) 'ha_impl_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_impl_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_impl_unittest.Tpo $(DEPDIR)/ha_unittests-ha_impl_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_impl_unittest.cc' object='ha_unittests-ha_impl_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_impl_unittest.obj `if test -f 'ha_impl_unittest.cc'; then $(CYGPATH_W) 'ha_impl_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_impl_unittest.cc'; fi`
+
+ha_unittests-ha_service_unittest.o: ha_service_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_service_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo -c -o ha_unittests-ha_service_unittest.o `test -f 'ha_service_unittest.cc' || echo '$(srcdir)/'`ha_service_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo $(DEPDIR)/ha_unittests-ha_service_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_service_unittest.cc' object='ha_unittests-ha_service_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_service_unittest.o `test -f 'ha_service_unittest.cc' || echo '$(srcdir)/'`ha_service_unittest.cc
+
+ha_unittests-ha_service_unittest.obj: ha_service_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_service_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo -c -o ha_unittests-ha_service_unittest.obj `if test -f 'ha_service_unittest.cc'; then $(CYGPATH_W) 'ha_service_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_service_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_service_unittest.Tpo $(DEPDIR)/ha_unittests-ha_service_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_service_unittest.cc' object='ha_unittests-ha_service_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_service_unittest.obj `if test -f 'ha_service_unittest.cc'; then $(CYGPATH_W) 'ha_service_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_service_unittest.cc'; fi`
+
+ha_unittests-ha_test.o: ha_test.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_test.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_test.Tpo -c -o ha_unittests-ha_test.o `test -f 'ha_test.cc' || echo '$(srcdir)/'`ha_test.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_test.Tpo $(DEPDIR)/ha_unittests-ha_test.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_test.cc' object='ha_unittests-ha_test.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_test.o `test -f 'ha_test.cc' || echo '$(srcdir)/'`ha_test.cc
+
+ha_unittests-ha_test.obj: ha_test.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_test.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_test.Tpo -c -o ha_unittests-ha_test.obj `if test -f 'ha_test.cc'; then $(CYGPATH_W) 'ha_test.cc'; else $(CYGPATH_W) '$(srcdir)/ha_test.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_test.Tpo $(DEPDIR)/ha_unittests-ha_test.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_test.cc' object='ha_unittests-ha_test.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_test.obj `if test -f 'ha_test.cc'; then $(CYGPATH_W) 'ha_test.cc'; else $(CYGPATH_W) '$(srcdir)/ha_test.cc'; fi`
+
+ha_unittests-ha_mt_unittest.o: ha_mt_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_mt_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo -c -o ha_unittests-ha_mt_unittest.o `test -f 'ha_mt_unittest.cc' || echo '$(srcdir)/'`ha_mt_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo $(DEPDIR)/ha_unittests-ha_mt_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_mt_unittest.cc' object='ha_unittests-ha_mt_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_mt_unittest.o `test -f 'ha_mt_unittest.cc' || echo '$(srcdir)/'`ha_mt_unittest.cc
+
+ha_unittests-ha_mt_unittest.obj: ha_mt_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-ha_mt_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo -c -o ha_unittests-ha_mt_unittest.obj `if test -f 'ha_mt_unittest.cc'; then $(CYGPATH_W) 'ha_mt_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_mt_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-ha_mt_unittest.Tpo $(DEPDIR)/ha_unittests-ha_mt_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ha_mt_unittest.cc' object='ha_unittests-ha_mt_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-ha_mt_unittest.obj `if test -f 'ha_mt_unittest.cc'; then $(CYGPATH_W) 'ha_mt_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/ha_mt_unittest.cc'; fi`
+
+ha_unittests-lease_update_backlog_unittest.o: lease_update_backlog_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-lease_update_backlog_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo -c -o ha_unittests-lease_update_backlog_unittest.o `test -f 'lease_update_backlog_unittest.cc' || echo '$(srcdir)/'`lease_update_backlog_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='lease_update_backlog_unittest.cc' object='ha_unittests-lease_update_backlog_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-lease_update_backlog_unittest.o `test -f 'lease_update_backlog_unittest.cc' || echo '$(srcdir)/'`lease_update_backlog_unittest.cc
+
+ha_unittests-lease_update_backlog_unittest.obj: lease_update_backlog_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-lease_update_backlog_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo -c -o ha_unittests-lease_update_backlog_unittest.obj `if test -f 'lease_update_backlog_unittest.cc'; then $(CYGPATH_W) 'lease_update_backlog_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/lease_update_backlog_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Tpo $(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='lease_update_backlog_unittest.cc' object='ha_unittests-lease_update_backlog_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-lease_update_backlog_unittest.obj `if test -f 'lease_update_backlog_unittest.cc'; then $(CYGPATH_W) 'lease_update_backlog_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/lease_update_backlog_unittest.cc'; fi`
+
+ha_unittests-query_filter_unittest.o: query_filter_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-query_filter_unittest.o -MD -MP -MF $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo -c -o ha_unittests-query_filter_unittest.o `test -f 'query_filter_unittest.cc' || echo '$(srcdir)/'`query_filter_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo $(DEPDIR)/ha_unittests-query_filter_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='query_filter_unittest.cc' object='ha_unittests-query_filter_unittest.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-query_filter_unittest.o `test -f 'query_filter_unittest.cc' || echo '$(srcdir)/'`query_filter_unittest.cc
+
+ha_unittests-query_filter_unittest.obj: query_filter_unittest.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-query_filter_unittest.obj -MD -MP -MF $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo -c -o ha_unittests-query_filter_unittest.obj `if test -f 'query_filter_unittest.cc'; then $(CYGPATH_W) 'query_filter_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/query_filter_unittest.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-query_filter_unittest.Tpo $(DEPDIR)/ha_unittests-query_filter_unittest.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='query_filter_unittest.cc' object='ha_unittests-query_filter_unittest.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-query_filter_unittest.obj `if test -f 'query_filter_unittest.cc'; then $(CYGPATH_W) 'query_filter_unittest.cc'; else $(CYGPATH_W) '$(srcdir)/query_filter_unittest.cc'; fi`
+
+ha_unittests-run_unittests.o: run_unittests.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-run_unittests.o -MD -MP -MF $(DEPDIR)/ha_unittests-run_unittests.Tpo -c -o ha_unittests-run_unittests.o `test -f 'run_unittests.cc' || echo '$(srcdir)/'`run_unittests.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-run_unittests.Tpo $(DEPDIR)/ha_unittests-run_unittests.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='run_unittests.cc' object='ha_unittests-run_unittests.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-run_unittests.o `test -f 'run_unittests.cc' || echo '$(srcdir)/'`run_unittests.cc
+
+ha_unittests-run_unittests.obj: run_unittests.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -MT ha_unittests-run_unittests.obj -MD -MP -MF $(DEPDIR)/ha_unittests-run_unittests.Tpo -c -o ha_unittests-run_unittests.obj `if test -f 'run_unittests.cc'; then $(CYGPATH_W) 'run_unittests.cc'; else $(CYGPATH_W) '$(srcdir)/run_unittests.cc'; fi`
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ha_unittests-run_unittests.Tpo $(DEPDIR)/ha_unittests-run_unittests.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='run_unittests.cc' object='ha_unittests-run_unittests.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ha_unittests_CPPFLAGS) $(CPPFLAGS) $(ha_unittests_CXXFLAGS) $(CXXFLAGS) -c -o ha_unittests-run_unittests.obj `if test -f 'run_unittests.cc'; then $(CYGPATH_W) 'run_unittests.cc'; else $(CYGPATH_W) '$(srcdir)/run_unittests.cc'; fi`
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+
+# This directory's subdirectories are mostly independent; you can cd
+# into them and run 'make' without going through this Makefile.
+# To change the values of 'make' variables: instead of editing Makefiles,
+# (1) if the variable is set in 'config.status', edit 'config.status'
+# (which will cause the Makefiles to be regenerated when you run 'make');
+# (2) otherwise, pass the desired values on the 'make' command line.
+$(am__recursive_targets):
+ @fail=; \
+ if $(am__make_keepgoing); then \
+ failcom='fail=yes'; \
+ else \
+ failcom='exit 1'; \
+ fi; \
+ dot_seen=no; \
+ target=`echo $@ | sed s/-recursive//`; \
+ case "$@" in \
+ distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \
+ *) list='$(SUBDIRS)' ;; \
+ esac; \
+ for subdir in $$list; do \
+ echo "Making $$target in $$subdir"; \
+ if test "$$subdir" = "."; then \
+ dot_seen=yes; \
+ local_target="$$target-am"; \
+ else \
+ local_target="$$target"; \
+ fi; \
+ ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \
+ || eval $$failcom; \
+ done; \
+ if test "$$dot_seen" = "no"; then \
+ $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \
+ fi; test -z "$$fail"
+
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-recursive
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ set x; \
+ here=`pwd`; \
+ if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \
+ include_option=--etags-include; \
+ empty_fix=.; \
+ else \
+ include_option=--include; \
+ empty_fix=; \
+ fi; \
+ list='$(SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ test ! -f $$subdir/TAGS || \
+ set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \
+ fi; \
+ done; \
+ $(am__define_uniq_tagged_files); \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: ctags-recursive
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-recursive
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+check-TESTS: $(TESTS)
+ @failed=0; all=0; xfail=0; xpass=0; skip=0; \
+ srcdir=$(srcdir); export srcdir; \
+ list=' $(TESTS) '; \
+ $(am__tty_colors); \
+ if test -n "$$list"; then \
+ for tst in $$list; do \
+ if test -f ./$$tst; then dir=./; \
+ elif test -f $$tst; then dir=; \
+ else dir="$(srcdir)/"; fi; \
+ if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \
+ all=`expr $$all + 1`; \
+ case " $(XFAIL_TESTS) " in \
+ *[\ \ ]$$tst[\ \ ]*) \
+ xpass=`expr $$xpass + 1`; \
+ failed=`expr $$failed + 1`; \
+ col=$$red; res=XPASS; \
+ ;; \
+ *) \
+ col=$$grn; res=PASS; \
+ ;; \
+ esac; \
+ elif test $$? -ne 77; then \
+ all=`expr $$all + 1`; \
+ case " $(XFAIL_TESTS) " in \
+ *[\ \ ]$$tst[\ \ ]*) \
+ xfail=`expr $$xfail + 1`; \
+ col=$$lgn; res=XFAIL; \
+ ;; \
+ *) \
+ failed=`expr $$failed + 1`; \
+ col=$$red; res=FAIL; \
+ ;; \
+ esac; \
+ else \
+ skip=`expr $$skip + 1`; \
+ col=$$blu; res=SKIP; \
+ fi; \
+ echo "$${col}$$res$${std}: $$tst"; \
+ done; \
+ if test "$$all" -eq 1; then \
+ tests="test"; \
+ All=""; \
+ else \
+ tests="tests"; \
+ All="All "; \
+ fi; \
+ if test "$$failed" -eq 0; then \
+ if test "$$xfail" -eq 0; then \
+ banner="$$All$$all $$tests passed"; \
+ else \
+ if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
+ banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
+ fi; \
+ else \
+ if test "$$xpass" -eq 0; then \
+ banner="$$failed of $$all $$tests failed"; \
+ else \
+ if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
+ banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
+ fi; \
+ fi; \
+ dashes="$$banner"; \
+ skipped=""; \
+ if test "$$skip" -ne 0; then \
+ if test "$$skip" -eq 1; then \
+ skipped="($$skip test was not run)"; \
+ else \
+ skipped="($$skip tests were not run)"; \
+ fi; \
+ test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
+ dashes="$$skipped"; \
+ fi; \
+ report=""; \
+ if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
+ report="Please report to $(PACKAGE_BUGREPORT)"; \
+ test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
+ dashes="$$report"; \
+ fi; \
+ dashes=`echo "$$dashes" | sed s/./=/g`; \
+ if test "$$failed" -eq 0; then \
+ col="$$grn"; \
+ else \
+ col="$$red"; \
+ fi; \
+ echo "$${col}$$dashes$${std}"; \
+ echo "$${col}$$banner$${std}"; \
+ test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \
+ test -z "$$report" || echo "$${col}$$report$${std}"; \
+ echo "$${col}$$dashes$${std}"; \
+ test "$$failed" -eq 0; \
+ else :; fi
+
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+ @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ $(am__make_dryrun) \
+ || test -d "$(distdir)/$$subdir" \
+ || $(MKDIR_P) "$(distdir)/$$subdir" \
+ || exit 1; \
+ dir1=$$subdir; dir2="$(distdir)/$$subdir"; \
+ $(am__relativize); \
+ new_distdir=$$reldir; \
+ dir1=$$subdir; dir2="$(top_distdir)"; \
+ $(am__relativize); \
+ new_top_distdir=$$reldir; \
+ echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \
+ echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \
+ ($(am__cd) $$subdir && \
+ $(MAKE) $(AM_MAKEFLAGS) \
+ top_distdir="$$new_top_distdir" \
+ distdir="$$new_distdir" \
+ am__remove_distdir=: \
+ am__skip_length_check=: \
+ am__skip_mode_fix=: \
+ distdir) \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+ $(MAKE) $(AM_MAKEFLAGS) check-TESTS
+check: check-recursive
+all-am: Makefile $(PROGRAMS)
+installdirs: installdirs-recursive
+installdirs-am:
+install: install-recursive
+install-exec: install-exec-recursive
+install-data: install-data-recursive
+uninstall: uninstall-recursive
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-recursive
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+ -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-recursive
+
+clean-am: clean-generic clean-libtool clean-noinstPROGRAMS \
+ mostlyclean-am
+
+distclean: distclean-recursive
+ -rm -f ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_test.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-run_unittests.Po
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-tags
+
+dvi: dvi-recursive
+
+dvi-am:
+
+html: html-recursive
+
+html-am:
+
+info: info-recursive
+
+info-am:
+
+install-data-am:
+
+install-dvi: install-dvi-recursive
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-recursive
+
+install-html-am:
+
+install-info: install-info-recursive
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-recursive
+
+install-pdf-am:
+
+install-ps: install-ps-recursive
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-recursive
+ -rm -f ./$(DEPDIR)/ha_unittests-command_creator_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-communication_state_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_config_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_impl_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_mt_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_service_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-ha_test.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-lease_update_backlog_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-query_filter_unittest.Po
+ -rm -f ./$(DEPDIR)/ha_unittests-run_unittests.Po
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-recursive
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-recursive
+
+pdf-am:
+
+ps: ps-recursive
+
+ps-am:
+
+uninstall-am:
+
+.MAKE: $(am__recursive_targets) check-am install-am install-strip
+
+.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \
+ am--depfiles check check-TESTS check-am clean clean-generic \
+ clean-libtool clean-noinstPROGRAMS cscopelist-am ctags \
+ ctags-am distclean distclean-compile distclean-generic \
+ distclean-libtool distclean-tags distdir dvi dvi-am html \
+ html-am info info-am install install-am install-data \
+ install-data-am install-dvi install-dvi-am install-exec \
+ install-exec-am install-html install-html-am install-info \
+ install-info-am install-man install-pdf install-pdf-am \
+ install-ps install-ps-am install-strip installcheck \
+ installcheck-am installdirs installdirs-am maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-compile \
+ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+ tags tags-am uninstall uninstall-am
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc
new file mode 100644
index 0000000..8966b4c
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc
@@ -0,0 +1,530 @@
+// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <ha_server_type.h>
+#include <lease_update_backlog.h>
+#include <command_creator.h>
+#include <asiolink/io_address.h>
+#include <cc/data.h>
+#include <exceptions/exceptions.h>
+#include <dhcp/hwaddr.h>
+#include <dhcpsrv/lease.h>
+#include <boost/pointer_cast.hpp>
+#include <gtest/gtest.h>
+#include <vector>
+
+using namespace isc;
+using namespace isc::asiolink;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::ha;
+
+namespace {
+
+/// @brief Creates lease instance used in tests.
+///
+/// It creates a lease with an address of 192.1.2.3 and for the HW address
+/// of 0b:0b:0b:0b:0b:0b.
+///
+/// @return Pointer to the created lease.
+Lease4Ptr createLease4() {
+ HWAddrPtr hwaddr(new HWAddr(std::vector<uint8_t>(6, 11), HTYPE_ETHER));
+ Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr,
+ static_cast<const uint8_t*>(0), 0,
+ 60, 0, 1));
+ return (lease4);
+}
+
+/// @brief Creates IPv6 lease instance used in tests.
+///
+/// It creates a lease with an address of 2001:db8:1::cafe and for the
+/// DUID of 02:02:02:02:02:02:02:02.
+///
+/// @return Pointer to the created lease.
+Lease6Ptr createLease6() {
+ DuidPtr duid(new DUID(std::vector<uint8_t>(8, 02)));
+ Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"),
+ duid, 1234, 50, 60, 1));
+ return (lease6);
+}
+
+/// @brief Returns JSON representation of the lease.
+///
+/// @param lease_ptr Pointer to the lease.
+/// @return Pointer to the JSON representation of the lease.
+ElementPtr leaseAsJson(const LeasePtr& lease_ptr) {
+ ElementPtr lease = boost::const_pointer_cast<Element>(lease_ptr->toElement());
+ // Replace cltt with expiration time as this is what the lease_cmds hooks
+ // library expects.
+ int64_t cltt = lease->get("cltt")->intValue();
+ int64_t valid_lifetime = lease->get("valid-lft")->intValue();
+ int64_t expire = cltt + valid_lifetime;
+ lease->set("expire", Element::create(expire));
+ lease->remove("cltt");
+ return (lease);
+}
+
+/// @brief Performs basic checks on the command.
+///
+/// It tests whether the command name is correct, if it contains arguments
+/// and if the arguments are held in a map.
+///
+/// @param command Pointer to the command to be tested.
+/// @param expected_command Expected command name.
+/// @param expected_service Expected service name.
+/// @param [out] arguments Pointer to the arguments map extracted from the
+/// provided command.
+void
+testCommandBasics(const ConstElementPtr& command,
+ const std::string& expected_command,
+ const std::string& expected_service,
+ ConstElementPtr& arguments) {
+ ASSERT_TRUE(command);
+ ASSERT_EQ(Element::map, command->getType());
+ ConstElementPtr command_name = command->get("command");
+ ASSERT_TRUE(command_name);
+
+ // Make sure the command is a string.
+ ASSERT_EQ(Element::string, command_name->getType());
+ // Verify the command name.
+ ASSERT_EQ(expected_command, command_name->stringValue());
+
+ // Make sure that the service is present and includes a single
+ // entry.
+ ConstElementPtr service = command->get("service");
+ ASSERT_TRUE(service);
+ ASSERT_EQ(Element::list, service->getType());
+ ASSERT_EQ(1, service->size());
+ ASSERT_EQ(Element::string, service->get(0)->getType());
+ EXPECT_EQ(expected_service, service->get(0)->stringValue());
+
+ // Make sure that arguments are present.
+ ConstElementPtr command_arguments = command->get("arguments");
+ ASSERT_TRUE(command_arguments);
+ // Make sure that arguments are held in a map.
+ ASSERT_EQ(Element::map, command_arguments->getType());
+
+ // Return extracted arguments.
+ arguments = command_arguments;
+}
+
+/// @brief Performs basic checks on the command.
+///
+/// This variant of the function expects no arguments to be provided.
+///
+/// @param command Pointer to the command to be tested.
+/// @param expected_command Expected command name.
+/// @param expected_service Expected service name.
+void
+testCommandBasics(const ConstElementPtr& command,
+ const std::string& expected_command,
+ const std::string& expected_service) {
+ ASSERT_TRUE(command);
+ ASSERT_EQ(Element::map, command->getType());
+ ConstElementPtr command_name = command->get("command");
+ ASSERT_TRUE(command_name);
+
+ // Make sure the command is a string.
+ ASSERT_EQ(Element::string, command_name->getType());
+ // Verify the command name.
+ ASSERT_EQ(expected_command, command_name->stringValue());
+
+ // Make sure that the service is present and includes a single
+ // entry.
+ ConstElementPtr service = command->get("service");
+ ASSERT_TRUE(service);
+ ASSERT_EQ(Element::list, service->getType());
+ ASSERT_EQ(1, service->size());
+ ASSERT_EQ(Element::string, service->get(0)->getType());
+ EXPECT_EQ(expected_service, service->get(0)->stringValue());
+
+ ConstElementPtr command_arguments = command->get("arguments");
+ ASSERT_FALSE(command_arguments);
+}
+
+// This test verifies that the dhcp-disable command is correct.
+TEST(CommandCreatorTest, createDHCPDisable4) {
+ // Create command with max-period value set to 20.
+ ConstElementPtr command = CommandCreator::createDHCPDisable(20, HAServerType::DHCPv4);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp4",
+ arguments));
+ ASSERT_EQ(2, arguments->size());
+ ConstElementPtr max_period = arguments->get("max-period");
+ ASSERT_TRUE(max_period);
+ ASSERT_EQ(Element::integer, max_period->getType());
+ EXPECT_EQ(20, max_period->intValue());
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+
+ // Repeat the test but this time the max-period is not specified.
+ command = CommandCreator::createDHCPDisable(0, HAServerType::DHCPv4);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp4",
+ arguments));
+ ASSERT_EQ(1, arguments->size());
+ origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+}
+
+// This test verifies that the dhcp-enable command is correct.
+TEST(CommandCreatorTest, createDHCPEnable4) {
+ ConstElementPtr arguments;
+ ConstElementPtr command = CommandCreator::createDHCPEnable(HAServerType::DHCPv4);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-enable", "dhcp4",
+ arguments));
+ ASSERT_EQ(1, arguments->size());
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+}
+
+// This test verifies that the ha-reset command sent to DHCPv4 server is correct.
+TEST(CommandCreatorTest, createHAReset4) {
+ ConstElementPtr command = CommandCreator::createHAReset(HAServerType::DHCPv4);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-reset", "dhcp4"));
+}
+
+// This test verifies that the ha-heartbeat command is correct.
+TEST(CommandCreatorTest, createHeartbeat4) {
+ ConstElementPtr command = CommandCreator::createHeartbeat(HAServerType::DHCPv4);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-heartbeat", "dhcp4"));
+}
+
+// This test verifies that the command generated for the lease update
+// is correct.
+TEST(CommandCreatorTest, createLease4Update) {
+ ConstElementPtr command = CommandCreator::createLease4Update(*createLease4());
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-update", "dhcp4",
+ arguments));
+ ElementPtr lease_as_json = leaseAsJson(createLease4());
+ // The lease update must contain the "force-create" parameter indicating that
+ // the lease must be created if it doesn't exist.
+ lease_as_json->set("force-create", Element::create(true));
+ lease_as_json->set("origin", Element::create("ha-partner"));
+ EXPECT_EQ(lease_as_json->str(), arguments->str());
+}
+
+// This test verifies that the command generated for the lease deletion
+// is correct.
+TEST(CommandCreatorTest, createLease4Delete) {
+ ConstElementPtr command = CommandCreator::createLease4Delete(*createLease4());
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-del", "dhcp4",
+ arguments));
+ ElementPtr lease_as_json = leaseAsJson(createLease4());
+ lease_as_json->set("origin", Element::create("ha-partner"));
+ EXPECT_EQ(lease_as_json->str(), arguments->str());
+}
+
+// This test verifies that the lease4-get-all command is correct.
+TEST(CommandCreatorTest, createLease4GetAll) {
+ ConstElementPtr command = CommandCreator::createLease4GetAll();
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-all", "dhcp4"));
+}
+
+// This test verifies that the lease4-get-page command is correct when
+// first page is fetched.
+TEST(CommandCreatorTest, createLease4GetPageStart) {
+ Lease4Ptr lease4;
+ ConstElementPtr command = CommandCreator::createLease4GetPage(lease4, 10);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-page", "dhcp4",
+ arguments));
+
+ ConstElementPtr from = arguments->get("from");
+ ASSERT_TRUE(from);
+ EXPECT_EQ(Element::string, from->getType());
+ EXPECT_EQ("start", from->stringValue());
+
+ ConstElementPtr limit = arguments->get("limit");
+ ASSERT_TRUE(limit);
+ ASSERT_EQ(Element::integer, limit->getType());
+ EXPECT_EQ(10, limit->intValue());
+}
+
+// This test verifies that the lease4-get-page command is correct when next
+// page is fetched.
+TEST(CommandCreatorTest, createLease4GetPageAddress) {
+ Lease4Ptr lease4(new Lease4());
+ lease4->addr_ = IOAddress("1.2.3.4");
+
+ ConstElementPtr command = CommandCreator::createLease4GetPage(lease4, 15);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease4-get-page", "dhcp4",
+ arguments));
+
+ ConstElementPtr from = arguments->get("from");
+ ASSERT_TRUE(from);
+ EXPECT_EQ(Element::string, from->getType());
+ EXPECT_EQ("1.2.3.4", from->stringValue());
+
+ ConstElementPtr limit = arguments->get("limit");
+ ASSERT_TRUE(limit);
+ ASSERT_EQ(Element::integer, limit->getType());
+ EXPECT_EQ(15, limit->intValue());
+}
+
+// This test verifies that exception is thrown if limit is set to 0 while
+// creating lease4-get-page command.
+TEST(CommandCreatorTest, createLease4GetPageZeroLimit) {
+ Lease4Ptr lease4;
+ EXPECT_THROW(CommandCreator::createLease4GetPage(lease4, 0), BadValue);
+}
+
+// This test verifies that the dhcp-disable command (DHCPv6 case) is
+// correct.
+TEST(CommandCreatorTest, createDHCPDisable6) {
+ // Create command with max-period value set to 20.
+ ConstElementPtr command = CommandCreator::createDHCPDisable(20, HAServerType::DHCPv6);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp6",
+ arguments));
+ ASSERT_EQ(2, arguments->size());
+ ConstElementPtr max_period = arguments->get("max-period");
+ ASSERT_TRUE(max_period);
+ ASSERT_EQ(Element::integer, max_period->getType());
+ EXPECT_EQ(20, max_period->intValue());
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+
+ // Repeat the test but this time the max-period is not specified.
+ command = CommandCreator::createDHCPDisable(0, HAServerType::DHCPv6);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-disable", "dhcp6",
+ arguments));
+ ASSERT_EQ(1, arguments->size());
+ origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+}
+
+// This test verifies that the dhcp-enable command (DHCPv6 case) is
+// correct.
+TEST(CommandCreatorTest, createDHCPEnable6) {
+ ConstElementPtr arguments;
+ ConstElementPtr command = CommandCreator::createDHCPEnable(HAServerType::DHCPv6);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "dhcp-enable", "dhcp6",
+ arguments));
+ ASSERT_EQ(1, arguments->size());
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+}
+
+// This test verifies that the ha-reset command sent to DHCPv6 server is correct.
+TEST(CommandCreatorTest, createHAReset6) {
+ ConstElementPtr command = CommandCreator::createHAReset(HAServerType::DHCPv6);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-reset", "dhcp6"));
+}
+
+// This test verifies that the command generated for the lease update
+// is correct.
+TEST(CommandCreatorTest, createLease6Update) {
+ ConstElementPtr command = CommandCreator::createLease6Update(*createLease6());
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-update", "dhcp6",
+ arguments));
+ ElementPtr lease_as_json = leaseAsJson(createLease6());
+ // The lease update must contain the "force-create" parameter indicating that
+ // the lease must be created if it doesn't exist.
+ lease_as_json->set("force-create", Element::create(true));
+ lease_as_json->set("origin", Element::create("ha-partner"));
+ EXPECT_EQ(lease_as_json->str(), arguments->str());
+}
+
+// This test verifies that the command generated for the lease deletion
+// is correct.
+TEST(CommandCreatorTest, createLease6Delete) {
+ ConstElementPtr command = CommandCreator::createLease6Delete(*createLease6());
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-del", "dhcp6",
+ arguments));
+ ElementPtr lease_as_json = leaseAsJson(createLease6());
+ lease_as_json->set("origin", Element::create("ha-partner"));
+ EXPECT_EQ(lease_as_json->str(), arguments->str());
+}
+
+// This test verifies that the lease6-bulk-apply command is correct.
+TEST(CommandCreatorTest, createLease6BulkApply) {
+ Lease6Ptr lease = createLease6();
+ Lease6Ptr deleted_lease = createLease6();
+
+ Lease6CollectionPtr leases(new Lease6Collection());
+ Lease6CollectionPtr deleted_leases(new Lease6Collection());
+
+ leases->push_back(lease);
+ deleted_leases->push_back(lease);
+
+ ConstElementPtr command = CommandCreator::createLease6BulkApply(leases, deleted_leases);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-bulk-apply",
+ "dhcp6", arguments));
+
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+
+ // Verify deleted-leases.
+ auto deleted_leases_json = arguments->get("deleted-leases");
+ ASSERT_TRUE(deleted_leases_json);
+ ASSERT_EQ(Element::list, deleted_leases_json->getType());
+ ASSERT_EQ(1, deleted_leases_json->size());
+ auto lease_as_json = deleted_leases_json->get(0);
+ EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str());
+
+ // Verify leases.
+ auto leases_json = arguments->get("leases");
+ ASSERT_TRUE(leases_json);
+ ASSERT_EQ(Element::list, leases_json->getType());
+ ASSERT_EQ(1, leases_json->size());
+ lease_as_json = leases_json->get(0);
+ EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str());
+}
+
+// This test verifies that the lease6-bulk-apply command can be created
+// from DHCPv6 leases backlog.
+TEST(CommandCreatorTest, createLease6BulkApplyFromBacklog) {
+ Lease6Ptr lease = createLease6();
+ Lease6Ptr deleted_lease = createLease6();
+
+ LeaseUpdateBacklog backlog(100);
+ backlog.push(LeaseUpdateBacklog::ADD, lease);
+ backlog.push(LeaseUpdateBacklog::DELETE, deleted_lease);
+
+ ConstElementPtr command = CommandCreator::createLease6BulkApply(backlog);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-bulk-apply",
+ "dhcp6", arguments));
+
+ ConstElementPtr origin = arguments->get("origin");
+ ASSERT_TRUE(origin);
+ ASSERT_EQ("ha-partner", origin->stringValue());
+
+ // Verify deleted-leases.
+ auto deleted_leases_json = arguments->get("deleted-leases");
+ ASSERT_TRUE(deleted_leases_json);
+ ASSERT_EQ(Element::list, deleted_leases_json->getType());
+ ASSERT_EQ(1, deleted_leases_json->size());
+ auto lease_as_json = deleted_leases_json->get(0);
+ EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str());
+
+ // Verify leases.
+ auto leases_json = arguments->get("leases");
+ ASSERT_TRUE(leases_json);
+ ASSERT_EQ(Element::list, leases_json->getType());
+ ASSERT_EQ(1, leases_json->size());
+ lease_as_json = leases_json->get(0);
+ EXPECT_EQ(leaseAsJson(createLease6())->str(), lease_as_json->str());
+
+ // Make sure the backlog is now empty.
+ EXPECT_EQ(0, backlog.size());
+}
+
+// This test verifies that the lease6-get-all command is correct.
+TEST(CommandCreatorTest, createLease6GetAll) {
+ ConstElementPtr command = CommandCreator::createLease6GetAll();
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-all", "dhcp6"));
+}
+
+
+// This test verifies that the lease6-get-page command is correct when
+// first page is fetched.
+TEST(CommandCreatorTest, createLease6GetPageStart) {
+ Lease6Ptr lease6;
+ ConstElementPtr command = CommandCreator::createLease6GetPage(lease6, 10);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-page", "dhcp6",
+ arguments));
+
+ ConstElementPtr from = arguments->get("from");
+ ASSERT_TRUE(from);
+ EXPECT_EQ(Element::string, from->getType());
+ EXPECT_EQ("start", from->stringValue());
+
+ ConstElementPtr limit = arguments->get("limit");
+ ASSERT_TRUE(limit);
+ ASSERT_EQ(Element::integer, limit->getType());
+ EXPECT_EQ(10, limit->intValue());
+}
+
+// This test verifies that the lease6-get-page command is correct when next
+// page is fetched.
+TEST(CommandCreatorTest, createLease6GetPageAddress) {
+ Lease6Ptr lease6(new Lease6());
+ lease6->addr_ = IOAddress("2001:db8:1::1");
+
+ ConstElementPtr command = CommandCreator::createLease6GetPage(lease6, 15);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "lease6-get-page", "dhcp6",
+ arguments));
+
+ ConstElementPtr from = arguments->get("from");
+ ASSERT_TRUE(from);
+ EXPECT_EQ(Element::string, from->getType());
+ EXPECT_EQ("2001:db8:1::1", from->stringValue());
+
+ ConstElementPtr limit = arguments->get("limit");
+ ASSERT_TRUE(limit);
+ ASSERT_EQ(Element::integer, limit->getType());
+ EXPECT_EQ(15, limit->intValue());
+}
+
+// This test verifies that exception is thrown if limit is set to 0 while
+// creating lease6-get-page command.
+TEST(CommandCreatorTest, createLease6GetPageZeroLimit) {
+ Lease6Ptr lease6;
+ EXPECT_THROW(CommandCreator::createLease6GetPage(lease6, 0), BadValue);
+}
+
+// This test verifies that the ha-maintenance-notify command is correct
+// while being sent to the DHCPv4 server.
+TEST(CommandCreatorTest, createMaintenanceNotify4) {
+ ConstElementPtr command = CommandCreator::createMaintenanceNotify(true, HAServerType::DHCPv4);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp4",
+ arguments));
+
+ auto cancel = arguments->get("cancel");
+ ASSERT_TRUE(cancel);
+ ASSERT_EQ(Element::boolean, cancel->getType());
+ EXPECT_TRUE(cancel->boolValue());
+}
+
+// This test verifies that the ha-maintenance-notify command is correct
+// while being sent to the DHCPv6 server.
+TEST(CommandCreatorTest, createMaintenanceNotify6) {
+ ConstElementPtr command = CommandCreator::createMaintenanceNotify(false, HAServerType::DHCPv6);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp6",
+ arguments));
+
+ auto cancel = arguments->get("cancel");
+ ASSERT_TRUE(cancel);
+ ASSERT_EQ(Element::boolean, cancel->getType());
+ EXPECT_FALSE(cancel->boolValue());
+}
+
+// This test verifies that the ha-sync-complete-notify command sent to a
+// DHCPv4 server is correct.
+TEST(CommandCreatorTest, createSyncCompleteNotify4) {
+ ConstElementPtr command = CommandCreator::createSyncCompleteNotify(HAServerType::DHCPv4);
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-sync-complete-notify", "dhcp4"));
+}
+
+// This test verifies that the ha-sync-complete-notify command sent to a
+// DHCPv4 server is correct.
+TEST(CommandCreatorTest, createSyncCompleteNotify6) {
+ ConstElementPtr command = CommandCreator::createSyncCompleteNotify(HAServerType::DHCPv6);
+ ConstElementPtr arguments;
+ ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-sync-complete-notify", "dhcp6"));
+}
+
+}
diff --git a/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc
new file mode 100644
index 0000000..8c33c09
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc
@@ -0,0 +1,1201 @@
+// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <ha_test.h>
+#include <asiolink/asio_wrapper.h>
+#include <communication_state.h>
+#include <asiolink/interval_timer.h>
+#include <asiolink/io_service.h>
+#include <dhcp/dhcp4.h>
+#include <dhcp/dhcp6.h>
+#include <exceptions/exceptions.h>
+#include <http/date_time.h>
+#include <util/multi_threading_mgr.h>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <gtest/gtest.h>
+#include <functional>
+#include <limits>
+#include <sstream>
+
+using namespace isc;
+using namespace isc::asiolink;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::ha;
+using namespace isc::ha::test;
+using namespace isc::http;
+using namespace isc::util;
+
+using namespace boost::posix_time;
+using namespace boost::gregorian;
+
+
+namespace {
+
+
+/// @brief Test fixture class for @c CommunicationState class.
+class CommunicationStateTest : public HATest {
+public:
+
+ /// @brief Constructor.
+ CommunicationStateTest()
+ : state_(io_service_, createValidConfiguration()),
+ state6_(io_service_, createValidConfiguration()) {
+ MultiThreadingMgr::instance().setMode(false);
+ }
+
+ /// @brief Destructor.
+ ~CommunicationStateTest() {
+ MultiThreadingMgr::instance().setMode(false);
+ io_service_->poll();
+ }
+
+ /// @brief Verifies that the partner state is set and retrieved correctly.
+ void partnerStateTest();
+
+ /// @brief Verifies that the partner state is set to unavailable.
+ ///
+ /// Whether or not the function under test also resets the clock skew is
+ /// tested in a different test case.
+ void partnerStateUnavailableTest();
+
+ /// @brief Verifies that the partner's scopes are set and retrieved correctly.
+ void partnerScopesTest();
+
+ /// @brief Verifies that the object is poked right after construction.
+ void initialDurationTest();
+
+ /// @brief Verifies that poking the state updates the returned duration.
+ void pokeTest();
+
+ /// @brief Test that heartbeat function is triggered.
+ void heartbeatTest();
+
+ /// @brief Test that invalid values provided to startHeartbeat are rejected.
+ void startHeartbeatInvalidValuesTest();
+
+ /// @brief Test that failure detection works properly for DHCPv4 case.
+ void detectFailureV4Test();
+
+ /// @brief This test verifies that it is possible to disable analysis of the DHCPv4
+ /// packets in which case the partner's failure is assumed when there is
+ /// no connection over the control channel.
+ void failureDetectionDisabled4Test();
+
+ /// @brief Test that failure detection works properly for DHCPv6 case.
+ void detectFailureV6Test();
+
+ /// @brief This test verifies that it is possible to disable analysis of the DHCPv6
+ /// packets in which case the partner's failure is assumed when there is
+ /// no connection over the control channel.
+ void failureDetectionDisabled6Test();
+
+ /// @brief This test verifies that the clock skew is checked properly by the
+ /// clockSkewShouldWarn and clockSkewShouldTerminate functions.
+ void clockSkewTest();
+
+ /// @brief This test verifies that the clock skew calculations take into
+ /// account whether or not the partner is available.
+ void clockSkewPartnerUnavailableTest();
+
+ /// @brief This test verifies that the clock skew value is formatted correctly
+ /// for logging.
+ void logFormatClockSkewTest();
+
+ /// @brief This test verifies that too many rejected lease updates cause
+ /// the service termination.
+ void rejectedLeaseUpdatesTerminateTest();
+
+ /// @brief Tests that the communication state report is correct.
+ void getReportTest();
+
+ /// @brief Tests unusual values used to create the report.
+ void getReportDefaultValuesTest();
+
+ /// @brief Tests that unsent updates count can be incremented and fetched.
+ void getUnsentUpdateCountTest();
+
+ /// @brief Tests that unsent updates count from partner can be set and
+ /// a difference from previous value detected.
+ void hasPartnerNewUnsentUpdatesTest();
+
+ /// @brief Test that gathering rejected leases works fine in DHCPv4 case.
+ void reportRejectedLeasesV4Test();
+
+ /// @brief Test that rejected leases are cleared after reporting respective
+ /// successful leases.
+ void reportSuccessfulLeasesV4Test();
+
+ /// @brief Test that invalid values are not accepted when reporting
+ /// rejected leases.
+ void reportRejectedLeasesV4InvalidValuesTest();
+
+ /// @brief Test that gathering rejected leases works fine in the DHCPv6 case.
+ void reportRejectedLeasesV6Test();
+
+ /// @brief Test that rejected leases are cleared after reporting respective
+ /// successful leases.
+ void reportSuccessfulLeasesV6Test();
+
+ /// @brief Test that invalid values are not accepted when reporting
+ /// rejected leases.
+ void reportRejectedLeasesV6InvalidValuesTest();
+
+ /// @brief Test that old rejected lease updates are discarded while
+ /// getting the rejected lease updates count.
+ void getRejectedLeaseUpdatesCountFromContainerTest();
+
+ /// @brief Returns test heartbeat implementation.
+ ///
+ /// @return Pointer to heartbeat implementation function under test.
+ std::function<void()> getHeartbeatImpl() {
+ return (std::bind(&CommunicationStateTest::heartbeatImpl, this));
+ }
+
+ /// @brief Test heartbeat implementation.
+ ///
+ /// It simply pokes the communication state object. Note that the real
+ /// implementation would send an actual heartbeat command prior to
+ /// poking the state.
+ void heartbeatImpl() {
+ state_.poke();
+ }
+
+ /// @brief Communication state object used throughout the tests.
+ NakedCommunicationState4 state_;
+
+ /// @brief Communication state for IPv6 used throughout the tests.
+ NakedCommunicationState6 state6_;
+};
+
+// Verifies that the partner state is set and retrieved correctly.
+void
+CommunicationStateTest::partnerStateTest() {
+ // Initially the state is unknown.
+ EXPECT_LT(state_.getPartnerState(), 0);
+
+ state_.setPartnerState("hot-standby");
+ EXPECT_EQ(HA_HOT_STANDBY_ST, state_.getPartnerState());
+
+ state_.setPartnerState("load-balancing");
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, state_.getPartnerState());
+
+ state_.setPartnerState("partner-down");
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, state_.getPartnerState());
+
+ state_.setPartnerState("ready");
+ EXPECT_EQ(HA_READY_ST, state_.getPartnerState());
+
+ state_.setPartnerState("syncing");
+ EXPECT_EQ(HA_SYNCING_ST, state_.getPartnerState());
+
+ state_.setPartnerState("terminated");
+ EXPECT_EQ(HA_TERMINATED_ST, state_.getPartnerState());
+
+ state_.setPartnerState("waiting");
+ EXPECT_EQ(HA_WAITING_ST, state_.getPartnerState());
+
+ state_.setPartnerState("unavailable");
+ EXPECT_EQ(HA_UNAVAILABLE_ST, state_.getPartnerState());
+
+ // An attempt to set unsupported value should result in exception.
+ EXPECT_THROW(state_.setPartnerState("unsupported"), BadValue);
+}
+
+// Verifies that the partner state is set to unavailable.
+void
+CommunicationStateTest::partnerStateUnavailableTest() {
+ // Initially the state is unknown.
+ EXPECT_LT(state_.getPartnerState(), 0);
+
+ // Set a valid state initially.
+ state_.setPartnerState("hot-standby");
+ EXPECT_EQ(HA_HOT_STANDBY_ST, state_.getPartnerState());
+
+ state_.setPartnerUnavailable();
+ EXPECT_EQ(HA_UNAVAILABLE_ST, state_.getPartnerState());
+}
+
+// Verifies that the partner's scopes are set and retrieved correctly.
+void
+CommunicationStateTest::partnerScopesTest() {
+ // Initially, the scopes should be empty.
+ ASSERT_TRUE(state_.getPartnerScopes().empty());
+
+ // Set new partner scopes.
+ ASSERT_NO_THROW(
+ state_.setPartnerScopes(Element::fromJSON("[ \"server1\", \"server2\" ]"))
+ );
+
+ // Get them back.
+ auto returned = state_.getPartnerScopes();
+ EXPECT_EQ(2, returned.size());
+ EXPECT_EQ(1, returned.count("server1"));
+ EXPECT_EQ(1, returned.count("server2"));
+
+ // Override the scopes.
+ ASSERT_NO_THROW(
+ state_.setPartnerScopes(Element::fromJSON("[ \"server1\" ]"))
+ );
+ returned = state_.getPartnerScopes();
+ EXPECT_EQ(1, returned.size());
+ EXPECT_EQ(1, returned.count("server1"));
+
+ // Clear the scopes.
+ ASSERT_NO_THROW(
+ state_.setPartnerScopes(Element::fromJSON("[ ]"))
+ );
+ returned = state_.getPartnerScopes();
+ EXPECT_TRUE(returned.empty());
+
+ // An attempt to set invalid JSON should fail.
+ EXPECT_THROW(state_.setPartnerScopes(Element::fromJSON("{ \"not-a-list\": 1 }")),
+ BadValue);
+}
+
+// Verifies that the object is poked right after construction.
+void
+CommunicationStateTest::initialDurationTest() {
+ EXPECT_TRUE(state_.isPoked());
+}
+
+// Verifies that poking the state updates the returned duration.
+void
+CommunicationStateTest::pokeTest() {
+ state_.modifyPokeTime(-30);
+ ASSERT_GE(state_.getDurationInMillisecs(), 30000);
+ ASSERT_TRUE(state_.isCommunicationInterrupted());
+ ASSERT_NO_THROW(state_.poke());
+ EXPECT_TRUE(state_.isPoked());
+ EXPECT_FALSE(state_.isCommunicationInterrupted());
+}
+
+// Test that heartbeat function is triggered.
+void
+CommunicationStateTest::heartbeatTest() {
+ // Set poke time to the past and expect that the object is considered
+ // not poked.
+ state_.modifyPokeTime(-30);
+ EXPECT_FALSE(state_.isPoked());
+
+ // Run heartbeat every 1 second.
+ ASSERT_NO_THROW(state_.startHeartbeat(1, getHeartbeatImpl()));
+ runIOService(1200);
+
+ // After > than 1 second the state should have been poked.
+ EXPECT_TRUE(state_.isPoked());
+
+ // Repeat the test.
+ state_.modifyPokeTime(-30);
+ EXPECT_FALSE(state_.isPoked());
+ ASSERT_NO_THROW(state_.startHeartbeat(1, getHeartbeatImpl()));
+ runIOService(1200);
+ EXPECT_TRUE(state_.isPoked());
+}
+
+// Test that invalid values provided to startHeartbeat are rejected.
+void
+CommunicationStateTest::startHeartbeatInvalidValuesTest() {
+ EXPECT_THROW(state_.startHeartbeat(-1, getHeartbeatImpl()), BadValue);
+ EXPECT_THROW(state_.startHeartbeat(0, getHeartbeatImpl()), BadValue);
+ EXPECT_THROW(state_.startHeartbeat(1, 0), BadValue);
+}
+
+// Test that failure detection works properly for DHCPv4 case.
+void
+CommunicationStateTest::detectFailureV4Test() {
+ // Initially, there should be no unacked clients recorded.
+ ASSERT_FALSE(state_.failureDetected());
+ EXPECT_EQ(0, state_.getUnackedClientsCount());
+ EXPECT_EQ(0, state_.getConnectingClientsCount());
+ EXPECT_EQ(0, state_.getAnalyzedMessagesCount());
+
+ // The maximum number of unacked clients is 10. Let's provide 10
+ // DHCPDISCOVER messages with the "secs" value of 15 which exceeds
+ // the threshold of 10. All these clients should be recorded as
+ // unacked.
+ for (uint8_t i = 0; i < 10; ++i) {
+ // Some of the requests have no client identifier to test that
+ // we don't fall over if the client identifier is null.
+ const uint8_t client_id_seed = (i < 5 ? i : 0);
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i,
+ client_id_seed,
+ 15)));
+ // We don't exceed the maximum of number of unacked clients so the
+ // partner failure shouldn't be reported.
+ ASSERT_FALSE(state_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i);
+ }
+ EXPECT_EQ(10, state_.getUnackedClientsCount());
+ EXPECT_EQ(10, state_.getConnectingClientsCount());
+ EXPECT_EQ(10, state_.getAnalyzedMessagesCount());
+
+ // Let's provide similar set of requests but this time the "secs" field is
+ // below the threshold. They should not be counted as failures. Also,
+ // all of these requests have client identifier.
+ for (uint8_t i = 0; i < 10; ++i) {
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i,
+ 9)));
+ ASSERT_FALSE(state_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i);
+ }
+ EXPECT_EQ(10, state_.getUnackedClientsCount());
+ EXPECT_EQ(15, state_.getConnectingClientsCount());
+ EXPECT_EQ(20, state_.getAnalyzedMessagesCount());
+
+ // Let's create a message from a new (not recorded yet) client with the
+ // "secs" field value below the threshold. It should not be counted as failure.
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 10, 10, 6)));
+
+ // Still no failure.
+ ASSERT_FALSE(state_.failureDetected());
+ EXPECT_EQ(10, state_.getUnackedClientsCount());
+ EXPECT_EQ(16, state_.getConnectingClientsCount());
+ EXPECT_EQ(21, state_.getAnalyzedMessagesCount());
+
+ // Let's repeat one of the requests which already have been recorded as
+ // unacked but with a greater value of "secs" field. This should not
+ // be counted because only new clients count.
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 3, 3, 20)));
+ ASSERT_FALSE(state_.failureDetected());
+ EXPECT_EQ(10, state_.getUnackedClientsCount());
+ EXPECT_EQ(16, state_.getConnectingClientsCount());
+ EXPECT_EQ(22, state_.getAnalyzedMessagesCount());
+
+ // This time let's simulate a client with a MAC address already recorded but
+ // with a client identifier. This should be counted as a new unacked request.
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 7, 7, 15)));
+ ASSERT_TRUE(state_.failureDetected());
+ EXPECT_EQ(11, state_.getUnackedClientsCount());
+ EXPECT_EQ(16, state_.getConnectingClientsCount());
+ EXPECT_EQ(23, state_.getAnalyzedMessagesCount());
+
+ // Poking should cause all counters to reset as it is an indication that the
+ // control connection has been re-established.
+ ASSERT_NO_THROW(state_.poke());
+
+ // We're back to no failure state.
+ EXPECT_FALSE(state_.failureDetected());
+ EXPECT_EQ(0, state_.getUnackedClientsCount());
+ EXPECT_EQ(0, state_.getConnectingClientsCount());
+ EXPECT_EQ(0, state_.getAnalyzedMessagesCount());
+
+ // Send 11 DHCPDISCOVER messages with the "secs" field bytes swapped. Swapping
+ // bytes was reported for some misbehaving Windows clients. The server should
+ // detect bytes swapping when second byte is 0 and the first byte is non-zero.
+ // However, the first byte is equal to 5 which is below our threshold so none
+ // of the requests below should count as unacked.
+ for (uint8_t i = 0; i < 11; ++i) {
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i,
+ 0x0500)));
+ ASSERT_FALSE(state_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i)
+ << " when testing swapped secs field bytes";
+ }
+ EXPECT_EQ(0, state_.getUnackedClientsCount());
+ EXPECT_EQ(11, state_.getConnectingClientsCount());
+ EXPECT_EQ(11, state_.getAnalyzedMessagesCount());
+
+ // Repeat the same test, but this time either the first byte exceeds the
+ // secs threshold or the second byte is non-zero. All should be counted
+ // as unacked.
+ for (uint8_t i = 0; i < 10; ++i) {
+ uint16_t secs = (i % 2 == 0 ? 0x0F00 : 0x0501);
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, i, i,
+ secs)));
+ ASSERT_FALSE(state_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i)
+ << " when testing swapped secs field bytes";
+ }
+
+ // This last message should cause the failure state.
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 11, 11,
+ 0x30)));
+ EXPECT_TRUE(state_.failureDetected());
+ EXPECT_EQ(11, state_.getUnackedClientsCount());
+ EXPECT_EQ(12, state_.getConnectingClientsCount());
+ EXPECT_EQ(22, state_.getAnalyzedMessagesCount());
+}
+
+// This test verifies that it is possible to disable analysis of the DHCPv4
+// packets in which case the partner's failure is assumed when there is
+// no connection over the control channel.
+void
+CommunicationStateTest::failureDetectionDisabled4Test() {
+ state_.config_->setMaxUnackedClients(0);
+ EXPECT_TRUE(state_.failureDetected());
+}
+
+// Test that failure detection works properly for DHCPv6 case.
+void
+CommunicationStateTest::detectFailureV6Test() {
+ // Initially, there should be no unacked clients recorded.
+ ASSERT_FALSE(state6_.failureDetected());
+ EXPECT_EQ(0, state6_.getUnackedClientsCount());
+ EXPECT_EQ(0, state6_.getConnectingClientsCount());
+ EXPECT_EQ(0, state6_.getAnalyzedMessagesCount());
+
+ // The maximum number of unacked clients is 10. Let's provide 10
+ // Solicit messages with the "elapsed time" value of 1500 which exceeds
+ // the threshold of 10000ms. Note that the elapsed time value is provided
+ // in 1/100s of 1 second. All these clients should be recorded as
+ // unacked.
+ for (uint8_t i = 0; i < 10; ++i) {
+ ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, i,
+ 1500)));
+ // We don't exceed the maximum number of unacked clients so the
+ // partner failure shouldn't be reported.
+ ASSERT_FALSE(state6_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i);
+ }
+ EXPECT_EQ(10, state6_.getUnackedClientsCount());
+ EXPECT_EQ(10, state6_.getConnectingClientsCount());
+ EXPECT_EQ(10, state6_.getAnalyzedMessagesCount());
+
+ // Let's provide similar set of requests but this time the "elapsed time" is
+ // below the threshold. This should not reduce the number of unacked or new
+ // clients.
+ for (uint8_t i = 0; i < 10; ++i) {
+ ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, i,
+ 900)));
+ ASSERT_FALSE(state6_.failureDetected())
+ << "failure detected for the request number "
+ << static_cast<int>(i);
+ }
+ EXPECT_EQ(10, state6_.getUnackedClientsCount());
+ EXPECT_EQ(10, state6_.getConnectingClientsCount());
+ EXPECT_EQ(20, state6_.getAnalyzedMessagesCount());
+
+ // Let's create a message from a new (not recorded yet) client with the
+ // "elapsed time" value below the threshold. It should not count as failure.
+ ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 10, 600)));
+
+ // Still no failure.
+ ASSERT_FALSE(state6_.failureDetected());
+ EXPECT_EQ(10, state6_.getUnackedClientsCount());
+ EXPECT_EQ(11, state6_.getConnectingClientsCount());
+ EXPECT_EQ(21, state6_.getAnalyzedMessagesCount());
+
+ // Let's repeat one of the requests which already have been recorded as
+ // unacked but with a greater value of "elapsed time". This should not
+ // be counted because only new clients count.
+ ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 3, 2000)));
+ ASSERT_FALSE(state6_.failureDetected());
+ EXPECT_EQ(10, state6_.getUnackedClientsCount());
+ EXPECT_EQ(11, state6_.getConnectingClientsCount());
+ EXPECT_EQ(22, state6_.getAnalyzedMessagesCount());
+
+ // New unacked client should cause failure to be detected.
+ ASSERT_NO_THROW(state6_.analyzeMessage(createMessage6(DHCPV6_SOLICIT, 11, 1500)));
+ ASSERT_TRUE(state6_.failureDetected());
+ EXPECT_EQ(11, state6_.getUnackedClientsCount());
+ EXPECT_EQ(12, state6_.getConnectingClientsCount());
+ EXPECT_EQ(23, state6_.getAnalyzedMessagesCount());
+
+ // Poking should cause all counters to reset as it is an indication that the
+ // control connection has been re-established.
+ ASSERT_NO_THROW(state6_.poke());
+
+ // We're back to no failure state.
+ EXPECT_FALSE(state6_.failureDetected());
+ EXPECT_EQ(0, state6_.getUnackedClientsCount());
+ EXPECT_EQ(0, state6_.getConnectingClientsCount());
+ EXPECT_EQ(0, state6_.getAnalyzedMessagesCount());
+}
+
+// This test verifies that it is possible to disable analysis of the DHCPv6
+// packets in which case the partner's failure is assumed when there is
+// no connection over the control channel.
+void
+CommunicationStateTest::failureDetectionDisabled6Test() {
+ state6_.config_->setMaxUnackedClients(0);
+ EXPECT_TRUE(state6_.failureDetected());
+}
+
+// This test verifies that the clock skew is checked properly by the
+// clockSkewShouldWarn and clockSkewShouldTerminate functions.
+void
+CommunicationStateTest::clockSkewTest() {
+ // Default clock skew is 0.
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+
+ // Partner time is ahead by 15s (no warning).
+ state_.clock_skew_ += boost::posix_time::time_duration(0, 0, 15);
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+
+ // Partner time is behind by 15s (no warning).
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ -= boost::posix_time::time_duration(0, 0, 15);
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+
+ // Partner time is ahead by 35s (warning).
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ += boost::posix_time::time_duration(0, 0, 35);
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+
+ // Partner time is behind by 35s (warning).
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ -= boost::posix_time::time_duration(0, 0, 35);
+ state_.last_clock_skew_warn_ = boost::posix_time::ptime();
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+
+ // Due to the gating mechanism this should not return true the second
+ // time.
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+
+ // But should warn if the warning was issued more than 60 seconds ago.
+ state_.last_clock_skew_warn_ -= boost::posix_time::time_duration(0, 1, 30);
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+
+ // Partner time is ahead by 65s (warning and terminate).
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ += boost::posix_time::time_duration(0, 1, 5);
+ state_.last_clock_skew_warn_ = boost::posix_time::ptime();
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+ EXPECT_TRUE(state_.clockSkewShouldTerminate());
+
+ // Partner time is behind by 65s (warning and terminate).
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ -= boost::posix_time::time_duration(0, 1, 5);
+ state_.last_clock_skew_warn_ = boost::posix_time::ptime();
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+ EXPECT_TRUE(state_.clockSkewShouldTerminate());
+}
+
+// This test verifies that the clock skew calculations take into
+// account whether or not the partner is available.
+void
+CommunicationStateTest::clockSkewPartnerUnavailableTest() {
+ // Default clock skew is 0.
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+
+ // Move the clock skew beyond the 60s limit. The alarms about the
+ // too high clock skew should be activated.
+ state_.setPartnerTime(HttpDateTime().rfc1123Format());
+ state_.clock_skew_ += boost::posix_time::time_duration(0, 1, 5);
+ EXPECT_TRUE(state_.clockSkewShouldWarn());
+ EXPECT_TRUE(state_.clockSkewShouldTerminate());
+
+ // Mark the partner as unavailable. It should reset the clock skew
+ // because we don't know the actual partner's time at the moment.
+ state_.setPartnerUnavailable();
+ EXPECT_FALSE(state_.clockSkewShouldWarn());
+ EXPECT_FALSE(state_.clockSkewShouldTerminate());
+}
+
+// This test verifies that the clock skew value is formatted correctly
+// for logging.
+void
+CommunicationStateTest::logFormatClockSkewTest() {
+ // Make sure logFormatClockSkew() does not throw if called prior
+ // the first call to setPartnerTime().
+ std::string log;
+ ASSERT_NO_THROW(log = state_.logFormatClockSkew());
+ EXPECT_EQ(std::string("skew not initialized"), log);
+
+ // Get current time.
+ boost::posix_time::ptime now = HttpDateTime().getPtime();
+
+ // Partner time is ahead by 15s.
+ boost::posix_time::time_duration offset(0,0,15);
+ state_.setPartnerTime(HttpDateTime(now + offset).rfc1123Format());
+ ASSERT_NO_THROW(log = state_.logFormatClockSkew());
+
+ // The logFormatClockSkew uses the clock_skew_ value which is computed
+ // at the time when setPartnerTime() is called. Therefore, we can't
+ // just assume that it is 15s because it may be already slightly off.
+ // Let's compare the output with the actual clock_skew_ value remembered
+ // in the state_ instance.
+ ASSERT_FALSE(state_.clock_skew_.is_special());
+ ASSERT_FALSE(state_.clock_skew_.is_negative());
+ std::ostringstream s;
+ s << state_.clock_skew_.seconds() << "s ahead";
+ EXPECT_TRUE(log.find(s.str()) != std::string::npos) <<
+ " log content wrong: " << log;
+
+ // Partner time is behind by 15s.
+ state_.setPartnerTime(HttpDateTime(now - offset).rfc1123Format());
+ ASSERT_NO_THROW(log = state_.logFormatClockSkew());
+
+ // Again, extract the actual clock skew remembered in the state_ instance.
+ ASSERT_FALSE(state_.clock_skew_.is_special());
+ auto skew = state_.clock_skew_;
+
+ // It must be negative this time.
+ ASSERT_TRUE(skew.is_negative());
+ // Convert it to positive value so we can use to to build the expected string.
+ skew = -skew;
+ std::ostringstream s2;
+ s2 << skew.seconds() << "s behind";
+ EXPECT_TRUE(log.find(s2.str()) != std::string::npos) <<
+ " log content wrong: " << log;
+
+ offset = hours(18) + minutes(37) + seconds(15);
+ ptime mytime(date(2019, Jul, 23), offset);
+
+ state_.my_time_at_skew_ = mytime;
+ state_.partner_time_at_skew_ = mytime + seconds(25);
+ state_.clock_skew_ = seconds(25);
+ ASSERT_NO_THROW(log = state_.logFormatClockSkew());
+ std::string expected("my time: 2019-07-23 18:37:15, "
+ "partner's time: 2019-07-23 18:37:40, "
+ "partner's clock is 25s ahead");
+ EXPECT_EQ(expected, log);
+}
+
+void
+CommunicationStateTest::rejectedLeaseUpdatesTerminateTest() {
+ EXPECT_FALSE(state_.rejectedLeaseUpdatesShouldTerminate());
+ // Reject several lease updates but do not exceed the limit.
+ for (auto i = 0; i < 9; ++i) {
+ ASSERT_NO_THROW(state_.reportRejectedLeaseUpdate(createMessage4(DHCPDISCOVER, i, i, 0)));
+ }
+ EXPECT_FALSE(state_.rejectedLeaseUpdatesShouldTerminate());
+ // Add one more. It should exceed the limit.
+ ASSERT_NO_THROW(state_.reportRejectedLeaseUpdate(createMessage4(DHCPDISCOVER, 9, 9, 0)));
+ EXPECT_TRUE(state_.rejectedLeaseUpdatesShouldTerminate());
+}
+
+// Tests that the communication state report is correct.
+void
+CommunicationStateTest::getReportTest() {
+ state_.setPartnerState("waiting");
+
+ auto scopes = Element::createList();
+ scopes->add(Element::create("server1"));
+ state_.setPartnerScopes(scopes);
+
+ state_.poke();
+
+ // Simulate the communications interrupted state.
+ state_.modifyPokeTime(-100);
+
+ // Send two DHCP packets of which one has secs value beyond the threshold and
+ // the other one lower than the threshold.
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 0, 0, 5)));
+ ASSERT_NO_THROW(state_.analyzeMessage(createMessage4(DHCPDISCOVER, 1, 0, 15)));
+
+ // Get the report.
+ auto report = state_.getReport();
+ ASSERT_TRUE(report);
+
+ // Compare with the expected output.
+ std::string expected = "{"
+ " \"age\": 100,"
+ " \"in-touch\": true,"
+ " \"last-scopes\": [ \"server1\" ],"
+ " \"last-state\": \"waiting\","
+ " \"communication-interrupted\": true,"
+ " \"connecting-clients\": 2,"
+ " \"unacked-clients\": 1,"
+ " \"unacked-clients-left\": 10,"
+ " \"analyzed-packets\": 2"
+ "}";
+ EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), report));
+}
+
+// Tests unusual values used to create the report.
+void
+CommunicationStateTest::getReportDefaultValuesTest() {
+ auto report = state_.getReport();
+ ASSERT_TRUE(report);
+
+ // Compare with the expected output.
+ std::string expected = "{"
+ " \"age\": 0,"
+ " \"in-touch\": false,"
+ " \"last-scopes\": [ ],"
+ " \"last-state\": \"\","
+ " \"communication-interrupted\": false,"
+ " \"connecting-clients\": 0,"
+ " \"unacked-clients\": 0,"
+ " \"unacked-clients-left\": 0,"
+ " \"analyzed-packets\": 0"
+ "}";
+ EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), report));
+}
+
+void
+CommunicationStateTest::getUnsentUpdateCountTest() {
+ // Initially the count should be 0.
+ EXPECT_EQ(0, state_.getUnsentUpdateCount());
+
+ // Increasing the value by 1 several times.
+ EXPECT_NO_THROW(state_.increaseUnsentUpdateCount());
+ EXPECT_EQ(1, state_.getUnsentUpdateCount());
+ EXPECT_NO_THROW(state_.increaseUnsentUpdateCount());
+ EXPECT_EQ(2, state_.getUnsentUpdateCount());
+ EXPECT_NO_THROW(state_.increaseUnsentUpdateCount());
+ EXPECT_EQ(3, state_.getUnsentUpdateCount());
+
+ // Test that the method under test protects against an overflow
+ // resetting the value to 0.
+ state_.unsent_update_count_ = std::numeric_limits<uint64_t>::max();
+ EXPECT_NO_THROW(state_.increaseUnsentUpdateCount());
+ EXPECT_EQ(1, state_.getUnsentUpdateCount());
+}
+
+void
+CommunicationStateTest::hasPartnerNewUnsentUpdatesTest() {
+ // Initially the counts should be 0.
+ EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates());
+
+ // Set a positive value. It should be noticed.
+ EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(5));
+ EXPECT_TRUE(state_.hasPartnerNewUnsentUpdates());
+
+ // No change, no new unsent updates.
+ EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(5));
+ EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates());
+
+ // Change it again. New updates.
+ EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(10));
+ EXPECT_TRUE(state_.hasPartnerNewUnsentUpdates());
+
+ // Set it to 0 to simulate restart. No updates.
+ EXPECT_NO_THROW(state_.setPartnerUnsentUpdateCount(0));
+ EXPECT_FALSE(state_.hasPartnerNewUnsentUpdates());
+}
+
+void
+CommunicationStateTest::reportRejectedLeasesV4Test() {
+ // Initially, there should be no rejected leases.
+ EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount());
+
+ // Reject lease update.
+ auto msg = createMessage4(DHCPREQUEST, 1, 0, 0);
+ state_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount());
+
+ // Reject another lease update.
+ msg = createMessage4(DHCPREQUEST, 2, 0, 0);
+ state_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount());
+
+ // Reject a lease with a short (zero) lease lifetime.
+ // This lease should be discarded when we call the
+ // getRejectedLeaseUpdatesCount().
+ msg = createMessage4(DHCPREQUEST, 3, 0, 0);
+ state_.reportRejectedLeaseUpdate(msg, 0);
+ EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount());
+
+ // Reject lease update for a client using the same MAC
+ // address but different client identifier. It should
+ // be treated as a different lease.
+ msg = createMessage4(DHCPREQUEST, 2, 1, 0);
+ state_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(3, state_.getRejectedLeaseUpdatesCount());
+
+ // Clear rejected leases and make sure the counter
+ // is now 0.
+ state_.clearRejectedLeaseUpdates();
+ EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount());
+}
+
+void
+CommunicationStateTest::reportSuccessfulLeasesV4Test() {
+ // Initially, there should be no rejected leases.
+ EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount());
+ auto msg0 = createMessage4(DHCPREQUEST, 1, 0, 0);
+ // Reject lease update.
+ state_.reportRejectedLeaseUpdate(msg0);
+ EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount());
+ // Reject another lease update.
+ auto msg1 = createMessage4(DHCPREQUEST, 2, 0, 0);
+ state_.reportRejectedLeaseUpdate(msg1);
+ EXPECT_EQ(2, state_.getRejectedLeaseUpdatesCount());
+ // Report successful lease for the first message.
+ // It should reduce the number of rejected lease
+ // updates.
+ EXPECT_TRUE(state_.reportSuccessfulLeaseUpdate(msg0));
+ EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount());
+ // Report successful lease update for another message.
+ auto msg2 = createMessage4(DHCPREQUEST, 1, 1, 0);
+ EXPECT_FALSE(state_.reportSuccessfulLeaseUpdate(msg2));
+ EXPECT_EQ(1, state_.getRejectedLeaseUpdatesCount());
+ // There should be no rejected lease updates.
+ EXPECT_TRUE(state_.reportSuccessfulLeaseUpdate(msg1));
+ EXPECT_EQ(0, state_.getRejectedLeaseUpdatesCount());
+}
+
+void
+CommunicationStateTest::reportRejectedLeasesV4InvalidValuesTest() {
+ // Populate one valid update. Without it our functions under test
+ // would return early.
+ state_.reportRejectedLeaseUpdate(createMessage4(DHCPREQUEST, 1, 0, 0));
+ // Using DHCPv6 message in the DHCPv4 context is a programming
+ // error and deserves an exception.
+ auto msg = createMessage6(DHCPV6_REQUEST, 1, 0);
+ EXPECT_THROW(state_.reportRejectedLeaseUpdate(msg), BadValue);
+ EXPECT_THROW(state_.reportSuccessfulLeaseUpdate(msg), BadValue);
+}
+
+void
+CommunicationStateTest::reportRejectedLeasesV6Test() {
+ // Initially, there should be no rejected leases.
+ EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount());
+
+ // Reject lease update.
+ auto msg = createMessage6(DHCPV6_REQUEST, 1, 0);
+ state6_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount());
+
+ // Reject another lease update.
+ msg = createMessage6(DHCPV6_REQUEST, 2, 0);
+ state6_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount());
+
+ // Reject a lease with a short (zero) lease lifetime.
+ // This lease should be discarded when we call the
+ // getRejectedLeaseUpdatesCount().
+ msg = createMessage6(DHCPV6_REQUEST, 3, 0);
+ state6_.reportRejectedLeaseUpdate(msg, 0);
+ EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount());
+
+ // Reject it again. It should not affect the counter.
+ msg = createMessage6(DHCPV6_REQUEST, 2, 0);
+ state6_.reportRejectedLeaseUpdate(msg);
+ EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount());
+
+ // Clear rejected lease updates and make sure they
+ // are now 0.
+ state6_.clearRejectedLeaseUpdates();
+ EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount());
+}
+
+void
+CommunicationStateTest::reportSuccessfulLeasesV6Test() {
+ // Initially, there should be no rejected leases.
+ EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount());
+ // Reject lease update.
+ auto msg0 = createMessage6(DHCPV6_SOLICIT, 1, 0);
+ EXPECT_TRUE(state6_.reportRejectedLeaseUpdate(msg0));
+ EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount());
+ // Reject another lease update.
+ auto msg1 = createMessage6(DHCPV6_SOLICIT, 2, 0);
+ EXPECT_TRUE(state6_.reportRejectedLeaseUpdate(msg1));
+ EXPECT_EQ(2, state6_.getRejectedLeaseUpdatesCount());
+ // Report successful lease for the first message.
+ // It should reduce the number of rejected lease
+ // updates.
+ EXPECT_TRUE(state6_.reportSuccessfulLeaseUpdate(msg0));
+ EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount());
+ // Report successful lease update for a lease that wasn't
+ // rejected. It should not affect the counter.
+ auto msg2 = createMessage6(DHCPV6_SOLICIT, 3, 0);
+ EXPECT_FALSE(state6_.reportSuccessfulLeaseUpdate(msg2));
+ EXPECT_EQ(1, state6_.getRejectedLeaseUpdatesCount());
+ // Report successful lease update for the last lease.
+ // The counter should now be 0.
+ EXPECT_TRUE(state6_.reportSuccessfulLeaseUpdate(msg1));
+ EXPECT_EQ(0, state6_.getRejectedLeaseUpdatesCount());
+}
+
+void
+CommunicationStateTest::reportRejectedLeasesV6InvalidValuesTest() {
+ // Populate one valid update. Without it our functions under test
+ // would return early.
+ state6_.reportRejectedLeaseUpdate(createMessage6(DHCPV6_REQUEST, 1, 0));
+ // Using DHCPv4 message in the DHCPv6 context is a programming
+ // error and deserves an exception.
+ auto msg0 = createMessage4(DHCPREQUEST, 1, 1, 0);
+ EXPECT_THROW(state6_.reportRejectedLeaseUpdate(msg0), BadValue);
+ EXPECT_THROW(state6_.reportSuccessfulLeaseUpdate(msg0), BadValue);
+
+ auto msg1 = createMessage6(DHCPV6_SOLICIT, 1, 0);
+ msg1->delOption(D6O_CLIENTID);
+ EXPECT_FALSE(state6_.reportRejectedLeaseUpdate(msg1));
+ EXPECT_FALSE(state6_.reportSuccessfulLeaseUpdate(msg1));
+}
+
+void
+CommunicationStateTest::getRejectedLeaseUpdatesCountFromContainerTest() {
+ // Create a simple multi index container with two indexes. The
+ // first index is on the ordinal number to distinguish between
+ // different entries. The second index is on the expire_ field
+ // that is identical to the expire_ field in the RejectedClients4
+ // and RejectedClients6 containers.
+ struct Entry {
+ int64_t ordinal_;
+ int64_t expire_;
+ };
+ typedef boost::multi_index_container<
+ Entry,
+ boost::multi_index::indexed_by<
+ boost::multi_index::ordered_unique<
+ boost::multi_index::member<Entry, int64_t,
+ &Entry::ordinal_>
+ >,
+ boost::multi_index::ordered_non_unique<
+ boost::multi_index::member<Entry, int64_t,
+ &Entry::expire_>
+ >
+ >
+ > Entries;
+ // Add many entries to the container. Odd entries have lifetime
+ // expiring in the future. Even entries have lifetimes expiring in
+ // the past.
+ Entries entries;
+ for (auto i = 0; i < 1000; i++) {
+ entries.insert({i, time(NULL) + (i % 2 ? 100 + i : -1 - i)});
+ }
+ // Get the count of valid entries. It should remove the expiring
+ // entries.
+ auto valid_entries_count = state_.getRejectedLeaseUpdatesCountFromContainer(entries);
+ EXPECT_EQ(500, valid_entries_count);
+ EXPECT_EQ(500, entries.size());
+
+ // Validate that we removed expired entries, not the valid ones.
+ for (auto entry : entries) {
+ EXPECT_EQ(1, entry.ordinal_ % 2);
+ }
+}
+
+TEST_F(CommunicationStateTest, partnerStateTest) {
+ partnerStateTest();
+}
+
+TEST_F(CommunicationStateTest, partnerStateTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ partnerStateTest();
+}
+
+TEST_F(CommunicationStateTest, partnerStateUnavailableTest) {
+ partnerStateUnavailableTest();
+}
+
+TEST_F(CommunicationStateTest, partnerStateUnavailableTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ partnerStateUnavailableTest();
+}
+
+TEST_F(CommunicationStateTest, partnerScopesTest) {
+ partnerScopesTest();
+}
+
+TEST_F(CommunicationStateTest, partnerScopesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ partnerScopesTest();
+}
+
+TEST_F(CommunicationStateTest, initialDurationTest) {
+ initialDurationTest();
+}
+
+TEST_F(CommunicationStateTest, initialDurationTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ initialDurationTest();
+}
+
+TEST_F(CommunicationStateTest, pokeTest) {
+ pokeTest();
+}
+
+TEST_F(CommunicationStateTest, pokeTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ pokeTest();
+}
+
+TEST_F(CommunicationStateTest, heartbeatTest) {
+ heartbeatTest();
+}
+
+TEST_F(CommunicationStateTest, heartbeatTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ heartbeatTest();
+}
+
+TEST_F(CommunicationStateTest, startHeartbeatInvalidValuesTest) {
+ startHeartbeatInvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, startHeartbeatInvalidValuesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ startHeartbeatInvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, detectFailureV4Test) {
+ detectFailureV4Test();
+}
+
+TEST_F(CommunicationStateTest, detectFailureV4TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ detectFailureV4Test();
+}
+
+TEST_F(CommunicationStateTest, failureDetectionDisabled4Test) {
+ failureDetectionDisabled4Test();
+}
+
+TEST_F(CommunicationStateTest, failureDetectionDisabled4TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ failureDetectionDisabled4Test();
+}
+
+TEST_F(CommunicationStateTest, detectFailureV6Test) {
+ detectFailureV6Test();
+}
+
+TEST_F(CommunicationStateTest, detectFailureV6TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ detectFailureV6Test();
+}
+
+TEST_F(CommunicationStateTest, failureDetectionDisabled6Test) {
+ failureDetectionDisabled6Test();
+}
+
+TEST_F(CommunicationStateTest, failureDetectionDisabled6TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ failureDetectionDisabled6Test();
+}
+
+TEST_F(CommunicationStateTest, clockSkewTest) {
+ clockSkewTest();
+}
+
+TEST_F(CommunicationStateTest, clockSkewTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ clockSkewTest();
+}
+
+TEST_F(CommunicationStateTest, clockSkewPartnerUnavailableTest) {
+ clockSkewPartnerUnavailableTest();
+}
+
+TEST_F(CommunicationStateTest, clockSkewPartnerUnavailableTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ clockSkewPartnerUnavailableTest();
+}
+
+TEST_F(CommunicationStateTest, rejectedLeaseUpdatesTerminateTest) {
+ rejectedLeaseUpdatesTerminateTest();
+}
+
+TEST_F(CommunicationStateTest, rejectedLeaseUpdatesTerminateTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ rejectedLeaseUpdatesTerminateTest();
+}
+
+TEST_F(CommunicationStateTest, logFormatClockSkewTest) {
+ logFormatClockSkewTest();
+}
+
+TEST_F(CommunicationStateTest, logFormatClockSkewTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ logFormatClockSkewTest();
+}
+
+TEST_F(CommunicationStateTest, getReportTest) {
+ getReportTest();
+}
+
+TEST_F(CommunicationStateTest, getReportTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ getReportTest();
+}
+
+TEST_F(CommunicationStateTest, getReportDefaultValuesTest) {
+ getReportDefaultValuesTest();
+}
+
+TEST_F(CommunicationStateTest, getReportDefaultValuesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ getReportDefaultValuesTest();
+}
+
+TEST_F(CommunicationStateTest, getUnsentUpdateCountTest) {
+ getUnsentUpdateCountTest();
+}
+
+TEST_F(CommunicationStateTest, getUnsentUpdateCountTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ getUnsentUpdateCountTest();
+}
+
+TEST_F(CommunicationStateTest, hasPartnerNewUnsentUpdatesTest) {
+ hasPartnerNewUnsentUpdatesTest();
+}
+
+TEST_F(CommunicationStateTest, hasPartnerNewUnsentUpdatesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hasPartnerNewUnsentUpdatesTest();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV4Test) {
+ reportRejectedLeasesV4Test();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV4TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportRejectedLeasesV4Test();
+}
+
+TEST_F(CommunicationStateTest, reportSuccessfulLeasesV4Test) {
+ reportSuccessfulLeasesV4Test();
+}
+
+TEST_F(CommunicationStateTest, reportSuccessfulLeasesV4TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportSuccessfulLeasesV4Test();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV4InvalidValuesTest) {
+ reportRejectedLeasesV4InvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV4InvalidValuesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportRejectedLeasesV4InvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV6Test) {
+ reportRejectedLeasesV6Test();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV6TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportRejectedLeasesV6Test();
+}
+
+TEST_F(CommunicationStateTest, reportSuccessfulLeasesV6Test) {
+ reportSuccessfulLeasesV6Test();
+}
+
+TEST_F(CommunicationStateTest, reportSuccessfulLeasesV6TestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportSuccessfulLeasesV6Test();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV6InvalidValuesTest) {
+ reportRejectedLeasesV6InvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, reportRejectedLeasesV6InvalidValuesTestMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ reportRejectedLeasesV6InvalidValuesTest();
+}
+
+TEST_F(CommunicationStateTest, getRejectedLeaseUpdatesCountFromContainerTest) {
+ getRejectedLeaseUpdatesCountFromContainerTest();
+}
+
+}
diff --git a/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc
new file mode 100644
index 0000000..f8e6940
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc
@@ -0,0 +1,1879 @@
+// Copyright (C) 2017-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <ha_impl.h>
+#include <ha_service_states.h>
+#include <ha_test.h>
+#include <cc/command_interpreter.h>
+#include <cc/data.h>
+#include <cc/dhcp_config_error.h>
+#include <config/command_mgr.h>
+#include <util/state_model.h>
+#include <util/multi_threading_mgr.h>
+#include <testutils/gtest_utils.h>
+#include <string>
+
+using namespace isc;
+using namespace isc::config;
+using namespace isc::data;
+using namespace isc::ha;
+using namespace isc::hooks;
+using namespace isc::ha::test;
+using namespace isc::util;
+
+namespace {
+
+/// @brief Test fixture class for testing HA hooks library
+/// configuration.
+class HAConfigTest : public HATest {
+public:
+ /// @brief Constructor.
+ HAConfigTest() : HATest(), hardware_threads_(MultiThreadingMgr::detectThreadCount()) {
+ }
+
+ /// @brief Verifies if an exception is thrown if provided HA
+ /// configuration is invalid.
+ ///
+ /// @param invalid_config Configuration to be tested.
+ /// @param expected_error Expected error message.
+ void testInvalidConfig(const std::string& invalid_config,
+ const std::string& expected_error) {
+ HAImplPtr impl(new HAImpl());
+ try {
+ impl->configure(Element::fromJSON(invalid_config));
+ ADD_FAILURE() << "expected ConfigError exception, thrown no exception";
+
+ } catch (const ConfigError& ex) {
+ // Expect the error to be contained in the exception message.
+ std::string const exception(ex.what());
+ EXPECT_NE(exception.find(expected_error), std::string::npos);
+ } catch (...) {
+ ADD_FAILURE() << "expected ConfigError exception, thrown different"
+ " exception type";
+ }
+ }
+
+ /// @brief number of threads the system reports as supported
+ uint32_t hardware_threads_;
+};
+
+// Verifies that load balancing configuration is parsed correctly.
+TEST_F(HAConfigTest, configureLoadBalancing) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"send-lease-updates\": false,"
+ " \"sync-leases\": false,"
+ " \"sync-timeout\": 20000,"
+ " \"sync-page-limit\": 3,"
+ " \"delayed-updates-limit\": 111,"
+ " \"heartbeat-delay\": 8,"
+ " \"max-response-delay\": 11,"
+ " \"max-ack-delay\": 5,"
+ " \"max-unacked-clients\": 20,"
+ " \"max-rejected-lease-updates\": 9,"
+ " \"wait-backup-ack\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"basic-auth-password\": \"1234\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"basic-auth-user\": \"\","
+ " \"role\": \"secondary\""
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:8082/\","
+ " \"basic-auth-user\": \"foo\","
+ " \"basic-auth-password\": \"bar\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " }"
+ " ],"
+ " \"state-machine\": {"
+ " \"states\": ["
+ " {"
+ " \"state\": \"waiting\","
+ " \"pause\": \"once\""
+ " },"
+ " {"
+ " \"state\": \"ready\","
+ " \"pause\": \"always\""
+ " },"
+ " {"
+ " \"state\": \"partner-down\","
+ " \"pause\": \"never\""
+ " }"
+ " ]"
+ " }"
+ " }"
+ "]";
+
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config)));
+ EXPECT_EQ("server1", impl->getConfig()->getThisServerName());
+ EXPECT_EQ(HAConfig::LOAD_BALANCING, impl->getConfig()->getHAMode());
+ EXPECT_FALSE(impl->getConfig()->amSendingLeaseUpdates());
+ EXPECT_FALSE(impl->getConfig()->amSyncingLeases());
+ EXPECT_EQ(20000, impl->getConfig()->getSyncTimeout());
+ EXPECT_EQ(3, impl->getConfig()->getSyncPageLimit());
+ EXPECT_EQ(111, impl->getConfig()->getDelayedUpdatesLimit());
+ EXPECT_TRUE(impl->getConfig()->amAllowingCommRecovery());
+ EXPECT_EQ(8, impl->getConfig()->getHeartbeatDelay());
+ EXPECT_EQ(11, impl->getConfig()->getMaxResponseDelay());
+ EXPECT_EQ(5, impl->getConfig()->getMaxAckDelay());
+ EXPECT_EQ(20, impl->getConfig()->getMaxUnackedClients());
+ EXPECT_EQ(9, impl->getConfig()->getMaxRejectedLeaseUpdates());
+ EXPECT_FALSE(impl->getConfig()->amWaitingBackupAck());
+
+ HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig();
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server1", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText());
+ EXPECT_EQ(cfg->getLogLabel(), "server1 (http://127.0.0.1:8080/)");
+ EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole());
+ EXPECT_FALSE(cfg->isAutoFailover());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ cfg = impl->getConfig()->getPeerConfig("server2");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server2", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText());
+ EXPECT_EQ(cfg->getLogLabel(), "server2 (http://127.0.0.1:8081/)");
+ EXPECT_EQ(HAConfig::PeerConfig::SECONDARY, cfg->getRole());
+ EXPECT_TRUE(cfg->isAutoFailover());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ cfg = impl->getConfig()->getPeerConfig("server3");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server3", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText());
+ EXPECT_EQ(cfg->getLogLabel(), "server3 (http://127.0.0.1:8082/)");
+ EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
+ EXPECT_FALSE(cfg->isAutoFailover());
+ ASSERT_TRUE(cfg->getBasicAuth());
+ EXPECT_EQ("foo:bar", cfg->getBasicAuth()->getSecret());
+
+ // Verify that per-state configuration is correct.
+
+ HAConfig::StateConfigPtr state_cfg;
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_BACKUP_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_COMMUNICATION_RECOVERY_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_LOAD_BALANCING_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_PARTNER_DOWN_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_READY_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_ALWAYS, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_SYNCING_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_TERMINATED_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_WAITING_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_ONCE, state_cfg->getPausing());
+
+ // Verify multi-threading default values. Default is 0 for the listener and client threads, but
+ // after MT is applied, HAImpl resolves them to the auto-detected values.
+ EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading());
+ EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads());
+}
+
+// Verifies that hot standby configuration is parsed correctly.
+TEST_F(HAConfigTest, configureHotStandby) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"basic-auth-user\": \"admin\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:8082/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " }"
+ " ]"
+ " }"
+ "]";
+
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config)));
+ EXPECT_EQ("server1", impl->getConfig()->getThisServerName());
+ EXPECT_EQ(HAConfig::HOT_STANDBY, impl->getConfig()->getHAMode());
+ EXPECT_TRUE(impl->getConfig()->amSendingLeaseUpdates());
+ EXPECT_TRUE(impl->getConfig()->amSyncingLeases());
+ EXPECT_EQ(60000, impl->getConfig()->getSyncTimeout());
+ EXPECT_EQ(10000, impl->getConfig()->getSyncPageLimit());
+ EXPECT_EQ(0, impl->getConfig()->getDelayedUpdatesLimit());
+ EXPECT_FALSE(impl->getConfig()->amAllowingCommRecovery());
+ EXPECT_EQ(10000, impl->getConfig()->getHeartbeatDelay());
+ EXPECT_EQ(10000, impl->getConfig()->getMaxAckDelay());
+ EXPECT_EQ(10, impl->getConfig()->getMaxUnackedClients());
+ EXPECT_EQ(10, impl->getConfig()->getMaxRejectedLeaseUpdates());
+ EXPECT_FALSE(impl->getConfig()->amWaitingBackupAck());
+
+ HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig();
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server1", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole());
+ EXPECT_FALSE(cfg->isAutoFailover());
+ ASSERT_TRUE(cfg->getBasicAuth());
+ EXPECT_EQ("admin:", cfg->getBasicAuth()->getSecret());
+
+ cfg = impl->getConfig()->getPeerConfig("server2");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server2", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::STANDBY, cfg->getRole());
+ EXPECT_TRUE(cfg->isAutoFailover());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ cfg = impl->getConfig()->getPeerConfig("server3");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server3", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
+ EXPECT_FALSE(cfg->isAutoFailover());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ HAConfig::StateConfigPtr state_cfg;
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_BACKUP_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_HOT_STANDBY_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_PARTNER_DOWN_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_READY_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_SYNCING_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_TERMINATED_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
+ getStateConfig(HA_WAITING_ST));
+ ASSERT_TRUE(state_cfg);
+ EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
+
+ // Verify multi-threading default values. Default is 0 for the listener and client threads, but
+ // after MT is applied, HAImpl resolves them to the auto-detected values.
+ EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading());
+ EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads());
+}
+
+// Verifies that passive-backup configuration is parsed correctly.
+TEST_F(HAConfigTest, configurePassiveBackup) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"wait-backup-ack\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"backup\""
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:8082/\","
+ " \"basic-auth-user\": \"keatest\","
+ " \"basic-auth-password-file\": \""
+ TEST_HTTP_DIR "/hiddenp\","
+ " \"role\": \"backup\""
+ " }"
+ " ]"
+ " }"
+ "]";
+
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW(impl->configure(Element::fromJSON(ha_config)));
+ EXPECT_EQ("server1", impl->getConfig()->getThisServerName());
+ EXPECT_EQ(HAConfig::PASSIVE_BACKUP, impl->getConfig()->getHAMode());
+ EXPECT_TRUE(impl->getConfig()->amSendingLeaseUpdates());
+ EXPECT_TRUE(impl->getConfig()->amWaitingBackupAck());
+
+ HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig();
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server1", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8080/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::PRIMARY, cfg->getRole());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ cfg = impl->getConfig()->getPeerConfig("server2");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server2", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8081/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
+ EXPECT_FALSE(cfg->getBasicAuth());
+
+ cfg = impl->getConfig()->getPeerConfig("server3");
+ ASSERT_TRUE(cfg);
+ EXPECT_EQ("server3", cfg->getName());
+ EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText());
+ EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
+ ASSERT_TRUE(cfg->getBasicAuth());
+ EXPECT_EQ("a2VhdGVzdDpLZWFUZXN0", cfg->getBasicAuth()->getCredential());
+
+ // Verify multi-threading default values. Default is 0 for the listener and client threads, but
+ // after MT is applied, HAImpl resolves them to the auto-detected values.
+ EXPECT_TRUE(impl->getConfig()->getEnableMultiThreading());
+ EXPECT_TRUE(impl->getConfig()->getHttpDedicatedListener());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpListenerThreads());
+ EXPECT_EQ(hardware_threads_, impl->getConfig()->getHttpClientThreads());
+}
+
+// This server name must not be empty.
+TEST_F(HAConfigTest, emptyServerName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'this-server-name' value must not be empty");
+}
+
+// There must be a configuration provided for this server.
+TEST_F(HAConfigTest, nonMatchingServerName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"foo\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "no peer configuration specified for the 'foo'");
+}
+
+// Error should be returned when mode is invalid.
+TEST_F(HAConfigTest, unsupportedMode) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"unsupported-mode\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "unsupported value 'unsupported-mode' for mode parameter");
+}
+
+// Error should be returned when heartbeat-delay is negative.
+TEST_F(HAConfigTest, negativeHeartbeatDelay) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"heartbeat-delay\": -1,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'heartbeat-delay' must not be negative");
+}
+
+// Error should be returned when heartbeat-delay is too large.
+TEST_F(HAConfigTest, largeHeartbeatDelay) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"heartbeat-delay\": 65536,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'heartbeat-delay' must not be greater than 65535");
+}
+
+// There must be at least two servers provided.
+TEST_F(HAConfigTest, singlePeer) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "secondary server required in the load balancing configuration");
+}
+
+// Server name must not be empty.
+TEST_F(HAConfigTest, emptyPeerName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "peer name must not be empty");
+}
+
+// Can't have two servers with the same name.
+TEST_F(HAConfigTest, duplicatePeerName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "peer with name 'server1' already specified");
+}
+
+// URL must be valid.
+TEST_F(HAConfigTest, invalidURL) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "invalid URL: url ://127.0.0.1:8080/ lacks http or"
+ " https scheme for server server2");
+}
+
+// URL hostname must be an address.
+TEST_F(HAConfigTest, badURLName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://localhost:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "bad url 'http://localhost:8080/': "
+ "Failed to convert string to address 'localhost': "
+ "Invalid argument for server server2");
+}
+
+// URL HTTPS scheme is not supported.
+TEST_F(HAConfigTest, badURLHttps) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"https://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "bad url 'https://127.0.0.1:8080/': "
+ "https scheme is not supported for server server2 "
+ "where TLS is disabled");
+}
+
+// Only certain roles are allowed.
+TEST_F(HAConfigTest, unsupportedRole) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"unsupported\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "unsupported value 'unsupported' for role parameter");
+}
+
+// There must be exactly one primary server.
+TEST_F(HAConfigTest, twoPrimary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "multiple primary servers specified");
+}
+
+// There must be exactly one secondary server.
+TEST_F(HAConfigTest, twoSecondary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "multiple secondary servers specified");
+}
+
+// Only one standby server is allowed.
+TEST_F(HAConfigTest, twoStandby) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "multiple standby servers specified");
+}
+
+// Primary server is required for load balancing.
+TEST_F(HAConfigTest, loadBalancingNoPrimary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "primary server required in the load balancing configuration");
+}
+
+// Secondary server is required for load balancing.
+TEST_F(HAConfigTest, loadBalancingNoSecondary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "secondary server required in the load balancing configuration");
+}
+
+// Primary server is required for hot standby mode.
+TEST_F(HAConfigTest, hotStandbyNoPrimary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "primary server required in the hot standby configuration");
+}
+
+// Standby server is required for hot standby mode.
+TEST_F(HAConfigTest, hotStandbyNoStandby) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "standby server required in the hot standby configuration");
+}
+
+// Standby server must not be specified in the load balancing mode.
+TEST_F(HAConfigTest, loadBalancingStandby) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "standby servers not allowed in the load balancing configuration");
+}
+
+// Secondary server must not be specified in the hot standby mode.
+TEST_F(HAConfigTest, hotStandbySecondary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "secondary servers not allowed in the hot standby configuration");
+}
+
+// state-machine parameter must be a map.
+TEST_F(HAConfigTest, invalidStateMachine) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ],"
+ " \"state-machine\": ["
+ " {"
+ " \"state\": \"foo\","
+ " \"pause\": \"always\""
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'state-machine' parameter must be a map");
+}
+
+// states within state-machine must be a list.
+TEST_F(HAConfigTest, invalidStatesList) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ],"
+ " \"state-machine\": {"
+ " \"states\": {"
+ " }"
+ " }"
+ " }"
+ "]",
+ "'states' parameter must be a list");
+}
+
+// State name must be recognized.
+TEST_F(HAConfigTest, invalidStateName) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ],"
+ " \"state-machine\": {"
+ " \"states\": ["
+ " {"
+ " \"state\": \"foo\","
+ " \"pause\": \"always\""
+ " }"
+ " ]"
+ " }"
+ " }"
+ "]",
+ "unknown state foo");
+}
+
+// Pause value must be recognized.
+TEST_F(HAConfigTest, invalidPauseValue) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ],"
+ " \"state-machine\": {"
+ " \"states\": ["
+ " {"
+ " \"state\": \"waiting\","
+ " \"pause\": \"foo\""
+ " }"
+ " ]"
+ " }"
+ " }"
+ "]",
+ "unsupported value foo of 'pause' parameter");
+}
+
+// Must not specify configuration for the same state twice.
+TEST_F(HAConfigTest, duplicatedStates) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ],"
+ " \"state-machine\": {"
+ " \"states\": ["
+ " {"
+ " \"state\": \"waiting\","
+ " \"pause\": \"always\""
+ " },"
+ " {"
+ " \"state\": \"ready\","
+ " \"pause\": \"always\""
+ " },"
+ " {"
+ " \"state\": \"waiting\","
+ " \"pause\": \"always\""
+ " }"
+ " ]"
+ " }"
+ " }"
+ "]",
+ "duplicated configuration for the 'waiting' state");
+}
+
+// Test that wait-backup-ack must not be enabled in the load-balancing
+// configuration.
+TEST_F(HAConfigTest, waitBackupAckLoadBalancing) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"wait-backup-ack\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'wait-backup-ack' must be set to false in the load balancing configuration");
+}
+
+// Test that wait-backup-ack must not be enabled in the hot-standby
+// configuration.
+TEST_F(HAConfigTest, waitBackupAckHotStandby) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"wait-backup-ack\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'wait-backup-ack' must be set to false in the hot standby configuration");
+}
+
+// Test that secondary server is not allowed in the passive-backup mode.
+TEST_F(HAConfigTest, passiveBackupSecondaryServer) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"secondary\""
+ " }"
+ " ]"
+ " }"
+ "]",
+ "secondary servers not allowed in the passive backup configuration");
+}
+
+// Test that standby server is not allowed in the passive-backup mode.
+TEST_F(HAConfigTest, passiveBackupStandbyServer) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"standby\""
+ " }"
+ " ]"
+ " }"
+ "]",
+ "standby servers not allowed in the passive backup configuration");
+}
+
+// Test that primary server is required in the passive-backup mode.
+TEST_F(HAConfigTest, passiveBackupNoPrimary) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]"
+ " }"
+ "]",
+ "primary server required in the passive backup configuration");
+}
+
+// Test that empty name id is forbidden for basic HTTP authentication.
+TEST_F(HAConfigTest, invalidUser) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"basic-auth-user\": \"foo:bar\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "user 'foo:bar' must not contain a ':' in peer 'server2'");
+}
+
+// Test that setting delayed-updates-limit is not allowed in hot-standby mode.
+TEST_F(HAConfigTest, hotStandbyDelayedUpdatesLimit) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"hot-standby\","
+ " \"delayed-updates-limit\": 1,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"standby\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'delayed-updates-limit' must be set to 0 in the hot standby configuration");
+}
+
+// Test that setting delayed-updates-limit is not allowed in passive-backup mode.
+TEST_F(HAConfigTest, passiveBackupDelayedUpdatesLimit) {
+ testInvalidConfig(
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"delayed-updates-limit\": 1,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]"
+ " }"
+ "]",
+ "'delayed-updates-limit' must be set to 0 in the passive backup configuration");
+}
+
+#if (defined(WITH_OPENSSL) || defined(WITH_BOTAN_BOOST))
+/// Test that TLS parameters are correctly inherited.
+TEST_F(HAConfigTest, tlsParameterInheritance) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"my-server\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"restrict-commands\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"my-server\","
+ " \"url\": \"https://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"overwrite\","
+ " \"trust-anchor\": \"!CA!\","
+ " \"cert-file\": \"!CA!/kea-server.crt\","
+ " \"key-file\": \"!CA!/kea-server.key\","
+ " \"url\": \"https://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"disable\","
+ " \"trust-anchor\": \"\","
+ " \"cert-file\": \"\","
+ " \"key-file\": \"\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW(impl->configure(Element::fromJSON(patched)));
+
+ // Check the global parameters.
+ std::string expected;
+ EXPECT_FALSE(impl->getConfig()->getTrustAnchor().unspecified());
+ expected = TEST_CA_DIR;
+ expected += "/kea-ca.crt";
+ EXPECT_EQ(expected, impl->getConfig()->getTrustAnchor().get());
+ EXPECT_FALSE(impl->getConfig()->getCertFile().unspecified());
+ expected = TEST_CA_DIR;
+ expected += "/kea-client.crt";
+ EXPECT_EQ(expected, impl->getConfig()->getCertFile().get());
+ EXPECT_FALSE(impl->getConfig()->getKeyFile().unspecified());
+ expected = TEST_CA_DIR;
+ expected += "/kea-client.key";
+ EXPECT_EQ(expected, impl->getConfig()->getKeyFile().get());
+ EXPECT_FALSE(impl->getConfig()->getRequireClientCerts());
+ EXPECT_TRUE(impl->getConfig()->getRestrictCommands());
+
+ // Check the first peer parameters: it inherits them from the global level.
+ HAConfig::PeerConfigPtr cfg = impl->getConfig()->getThisServerConfig();
+ ASSERT_TRUE(cfg);
+ EXPECT_TRUE(cfg->getTlsContext());
+
+ // Check the second peer parameters: it overwrites them.
+ cfg = impl->getConfig()->getPeerConfig("overwrite");
+ ASSERT_TRUE(cfg);
+ EXPECT_FALSE(cfg->getTrustAnchor().unspecified());
+ expected = TEST_CA_DIR;
+ EXPECT_EQ(expected, cfg->getTrustAnchor().get());
+ EXPECT_FALSE(cfg->getCertFile().unspecified());
+ expected = TEST_CA_DIR;
+ expected += "/kea-server.crt";
+ EXPECT_EQ(expected, cfg->getCertFile().get());
+ EXPECT_FALSE(cfg->getKeyFile().unspecified());
+ expected = TEST_CA_DIR;
+ expected += "/kea-server.key";
+ EXPECT_EQ(expected, cfg->getKeyFile().get());
+ EXPECT_TRUE(cfg->getTlsContext());
+
+ // Check the last peer parameters: it disables TLS by setting them to "".
+ cfg = impl->getConfig()->getPeerConfig("disable");
+ ASSERT_TRUE(cfg);
+ EXPECT_FALSE(cfg->getTrustAnchor().unspecified());
+ EXPECT_EQ("", cfg->getTrustAnchor().get());
+ EXPECT_FALSE(cfg->getCertFile().unspecified());
+ EXPECT_EQ("", cfg->getCertFile().get());
+ EXPECT_FALSE(cfg->getKeyFile().unspecified());
+ EXPECT_EQ("", cfg->getKeyFile().get());
+ // The TLS context should be null.
+ EXPECT_FALSE(cfg->getTlsContext());
+}
+
+// Test that a missing trust-anchor in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, missingTrustAnchor) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"trust-anchor\": \"\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server2: ";
+ expected += "trust-anchor parameter is missing or empty: ";
+ expected += "all or none of TLS parameters must be set";
+ testInvalidConfig(patched, expected);
+}
+
+// Test that a missing cert-file in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, missingCertFile) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"cert-file\": \"\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server2: ";
+ expected += "cert-file parameter is missing or empty: ";
+ expected += "all or none of TLS parameters must be set";
+ testInvalidConfig(patched, expected);
+}
+
+// Test that a missing key-file in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, missingKeyFile) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"key-file\": \"\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server2: ";
+ expected += "key-file parameter is missing or empty: ";
+ expected += "all or none of TLS parameters must be set";
+ testInvalidConfig(patched, expected);
+}
+
+// Test that a bad trust-anchor in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, badTrustAnchor) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"/this-file-does-not-exist\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server1: ";
+ expected += "load of CA file '/this-file-does-not-exist' failed: ";
+ // Backend dependent.
+#ifdef WITH_OPENSSL
+ expected += "No such file or directory";
+#else
+ expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist";
+#endif
+ testInvalidConfig(patched, expected);
+}
+
+// Test that a bad cert-file in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, badCertFile) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"/this-file-does-not-exist\","
+ " \"key-file\": \"!CA!/kea-client.key\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server1: ";
+ expected += "load of cert file '/this-file-does-not-exist' failed: ";
+ // Backend dependent.
+#ifdef WITH_OPENSSL
+ expected += "No such file or directory";
+#else
+ expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist";
+#endif
+ testInvalidConfig(patched, expected);
+}
+
+// Test that a bad key-file in the HTTPS parameter set raise an error.
+TEST_F(HAConfigTest, badKeyFile) {
+ const std::string ha_config =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"/this-file-does-not-exist\","
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": false"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " }"
+ " ]"
+ " }"
+ "]";
+ const std::string& patched = replaceInConfig(ha_config, "!CA!",
+ TEST_CA_DIR);
+ std::string expected = "bad TLS config for server server1: ";
+ expected += "load of private key file '/this-file-does-not-exist' failed: ";
+ // Backend dependent.
+#ifdef WITH_OPENSSL
+ expected += "No such file or directory";
+#else
+ expected += "I/O error: DataSource: Failure opening file /this-file-does-not-exist";
+#endif
+ testInvalidConfig(patched, expected);
+}
+#endif // WITH_OPENSSL || WITH_BOTAN_BOOST
+
+// Test that conversion of the role names works correctly.
+TEST_F(HAConfigTest, stringToRole) {
+ EXPECT_EQ(HAConfig::PeerConfig::PRIMARY,
+ HAConfig::PeerConfig::stringToRole("primary"));
+ EXPECT_EQ(HAConfig::PeerConfig::SECONDARY,
+ HAConfig::PeerConfig::stringToRole("secondary"));
+ EXPECT_EQ(HAConfig::PeerConfig::STANDBY,
+ HAConfig::PeerConfig::stringToRole("standby"));
+ EXPECT_EQ(HAConfig::PeerConfig::BACKUP,
+ HAConfig::PeerConfig::stringToRole("backup"));
+ EXPECT_THROW(HAConfig::PeerConfig::stringToRole("unsupported"),
+ BadValue);
+}
+
+// Test that role name is generated correctly.
+TEST_F(HAConfigTest, roleToString) {
+ EXPECT_EQ("primary",
+ HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::PRIMARY));
+ EXPECT_EQ("secondary",
+ HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::SECONDARY));
+ EXPECT_EQ("standby",
+ HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::STANDBY));
+ EXPECT_EQ("backup",
+ HAConfig::PeerConfig::roleToString(HAConfig::PeerConfig::BACKUP));
+}
+
+// Test that conversion of the HA mode names works correctly.
+TEST_F(HAConfigTest, stringToHAMode) {
+ EXPECT_EQ(HAConfig::LOAD_BALANCING, HAConfig::stringToHAMode("load-balancing"));
+ EXPECT_EQ(HAConfig::HOT_STANDBY, HAConfig::stringToHAMode("hot-standby"));
+ EXPECT_EQ(HAConfig::PASSIVE_BACKUP, HAConfig::stringToHAMode("passive-backup"));
+}
+
+// Test that HA mode name is generated correctly.
+TEST_F(HAConfigTest, HAModeToString) {
+ EXPECT_EQ("load-balancing", HAConfig::HAModeToString(HAConfig::LOAD_BALANCING));
+ EXPECT_EQ("hot-standby", HAConfig::HAModeToString(HAConfig::HOT_STANDBY));
+ EXPECT_EQ("passive-backup", HAConfig::HAModeToString(HAConfig::PASSIVE_BACKUP));
+}
+
+// Test that conversion of the 'pause' value works correctly.
+TEST_F(HAConfigTest, stringToPausing) {
+ EXPECT_EQ(STATE_PAUSE_ALWAYS,
+ HAConfig::StateConfig::stringToPausing("always"));
+ EXPECT_EQ(STATE_PAUSE_NEVER,
+ HAConfig::StateConfig::stringToPausing("never"));
+ EXPECT_EQ(STATE_PAUSE_ONCE,
+ HAConfig::StateConfig::stringToPausing("once"));
+}
+
+// Test that pause parameter value is generated correctly.
+TEST_F(HAConfigTest, pausingToString) {
+ EXPECT_EQ("always",
+ HAConfig::StateConfig::pausingToString(STATE_PAUSE_ALWAYS));
+ EXPECT_EQ("never",
+ HAConfig::StateConfig::pausingToString(STATE_PAUSE_NEVER));
+ EXPECT_EQ("once",
+ HAConfig::StateConfig::pausingToString(STATE_PAUSE_ONCE));
+}
+
+// Verifies permutations of HA+MT configuration.
+TEST_F(HAConfigTest, multiThreadingPermutations) {
+
+ // Structure describing a test scenario.
+ struct Scenario {
+ std::string desc_; // Description of the scenario.
+ std::string mt_json_; // multi-threading config to use.
+ bool dhcp_mt_enabled_; // True if DHCP multi-threading is enabled.
+ uint32_t dhcp_threads_; // Value of DHCP thread-pool-size.
+ bool exp_ha_mt_enabled_; // If HA+MT should be enabled
+ bool exp_listener_; // If HA+MT should use dedicated listener.
+ uint32_t exp_listener_threads_; // Expected number of listener threads.
+ uint32_t exp_client_threads_; // Expected number of client threads.
+ };
+
+ // Mnemonic constants.
+ bool dhcp_mt = true;
+ bool ha_mt = true;
+ bool listener = true;
+
+ std::vector<Scenario> scenarios {
+ {
+ "1 ha+mt by default",
+ "",
+ dhcp_mt, 4,
+ ha_mt, listener, 4, 4
+ },
+ {
+ "2 dhcp mt enabled, ha mt disabled",
+ makeHAMtJson(!ha_mt, !listener, 0, 0),
+ dhcp_mt, 4,
+ !ha_mt, !listener, 0, 0
+ },
+ {
+ "3 dhcp mt disabled, ha mt enabled",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ !dhcp_mt, 4,
+ !ha_mt, !listener, 0, 0
+ },
+ {
+ "4 dhcp mt enabled, ha mt enabled, listener disabled",
+ makeHAMtJson(ha_mt, !listener, 0, 0),
+ dhcp_mt, 4,
+ ha_mt, !listener, 4, 4
+ },
+ {
+ "5 dhcp mt enabled, ha mt enabled, listener enabled",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 4,
+ ha_mt, listener, 4, 4
+ },
+ {
+ "6 explicit DHCP threads, explicit thread values",
+ makeHAMtJson(ha_mt, listener, 5, 6),
+ dhcp_mt, 4,
+ ha_mt, listener, 5, 6
+ },
+ {
+ "7 explicit DHCP threads, zero thread values",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 8,
+ ha_mt, listener, 8, 8
+ },
+ {
+ "8 DHCP auto detect threads, zero thread values",
+ // Special case: if system reports supported threads as 0
+ // then HA+MT should be disabled. Otherwise it should
+ // be enabled with listener and client threads set to the
+ // reported value.
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 0,
+ (hardware_threads_ > 0), listener, hardware_threads_, hardware_threads_
+ }
+ };
+
+ // Iterate over the scenarios.
+ for (auto const& scenario : scenarios) {
+ SCOPED_TRACE(scenario.desc_);
+
+ // Build the HA JSON configuration.
+ std::stringstream ss;
+ ss <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"wait-backup-ack\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]";
+
+ if (!scenario.mt_json_.empty()) {
+ ss << "," << scenario.mt_json_;
+ }
+
+ ss << "}]";
+ ConstElementPtr config_json;
+ ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str()));
+
+ // Set DHCP multi-threading configuration in CfgMgr.
+ setDHCPMultiThreadingConfig(scenario.dhcp_mt_enabled_, scenario.dhcp_threads_);
+
+ // Create and configure the implementation.
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW_LOG(impl->configure(config_json));
+
+ // Fetch the updated config.
+ HAConfigPtr ha_config = impl->getConfig();
+
+ // Verify the configuration is as expected.
+ if (!scenario.exp_ha_mt_enabled_) {
+ // When HA+MT is disabled, the other values are moot.
+ ASSERT_FALSE(ha_config->getEnableMultiThreading());
+ } else {
+ ASSERT_TRUE(ha_config->getEnableMultiThreading());
+ EXPECT_EQ(ha_config->getHttpDedicatedListener(), scenario.exp_listener_);
+ EXPECT_EQ(ha_config->getHttpListenerThreads(), scenario.exp_listener_threads_);
+ EXPECT_EQ(ha_config->getHttpClientThreads(), scenario.exp_client_threads_);
+ }
+ }
+}
+
+// Check that an IPv6 address can be used as part of a value for "url".
+TEST_F(HAConfigTest, ipv6Url) {
+ std::string const ha_config(R"(
+ [
+ {
+ "mode": "load-balancing",
+ "peers": [
+ {
+ "name": "server1",
+ "role": "primary",
+ "url": "http://[2001:db8::1]:8080/"
+ },
+ {
+ "name": "server2",
+ "role": "secondary",
+ "url": "http://[2001:db8::2]:8080/"
+ }
+ ],
+ "this-server-name": "server1"
+ }
+ ]
+ )");
+
+ // Configure HA.
+ HAImplPtr impl(new HAImpl());
+ ASSERT_NO_THROW_LOG(impl->configure(Element::fromJSON(ha_config)));
+
+ // Check the URL.
+ EXPECT_EQ(impl->getConfig()->getThisServerConfig()->getUrl().toText(), "http://[2001:db8::1]:8080/");
+}
+
+} // namespace
diff --git a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc
new file mode 100644
index 0000000..9cfe46c
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc
@@ -0,0 +1,814 @@
+// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <ha_test.h>
+#include <ha_impl.h>
+#include <asiolink/io_address.h>
+#include <cc/command_interpreter.h>
+#include <cc/data.h>
+#include <dhcp/dhcp4.h>
+#include <dhcp/dhcp6.h>
+#include <dhcp/hwaddr.h>
+#include <dhcpsrv/lease.h>
+#include <dhcpsrv/network_state.h>
+#include <hooks/hooks_manager.h>
+#include <testutils/gtest_utils.h>
+#include <boost/pointer_cast.hpp>
+#include <gtest/gtest.h>
+#include <string>
+
+using namespace isc::asiolink;
+using namespace isc::config;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::ha;
+using namespace isc::ha::test;
+using namespace isc::hooks;
+
+namespace {
+
+/// @brief Structure that holds registered hook indexes.
+///
+/// This allows us to park packets.
+struct TestHooks {
+ /// @brief Index of leases4_committed callout.
+ int hook_index_leases4_committed_;
+
+ /// @brief Index of leases6_committed callout.
+ int hook_index_leases6_committed_;
+
+ /// @brief Constructor
+ ///
+ /// The constructor registers hook points for callout tests.
+ TestHooks() {
+ hook_index_leases4_committed_ =
+ HooksManager::registerHook("leases4_committed");
+ hook_index_leases6_committed_ =
+ HooksManager::registerHook("leases6_committed");
+ }
+};
+
+TestHooks test_hooks;
+
+/// @brief Derivation of the @c HAImpl which provides access to protected
+/// methods and members.
+class TestHAImpl : public HAImpl {
+public:
+
+ using HAImpl::config_;
+ using HAImpl::service_;
+};
+
+/// @brief Test fixture class for @c HAImpl.
+class HAImplTest : public HATest {
+public:
+
+ /// @brief Tests handler of a ha-sync command.
+ ///
+ /// It always expects that the error result is returned. The expected
+ /// error text should be provided as function argument.
+ ///
+ /// @param ha_sync_command command provided as text.
+ /// @param expected_response expected text response.
+ void testSynchronizeHandler(const std::string& ha_sync_command,
+ const std::string& expected_response) {
+ HAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ ConstElementPtr command = Element::fromJSON(ha_sync_command);
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ callout_handle->setArgument("command", command);
+
+ ASSERT_NO_THROW(ha_impl.synchronizeHandler(*callout_handle));
+
+ ConstElementPtr response;
+ callout_handle->getArgument("response", response);
+ ASSERT_TRUE(response);
+
+ checkAnswer(response, CONTROL_RESULT_ERROR, expected_response);
+ }
+};
+
+// Tests that HAService object is created for DHCPv4 service.
+TEST_F(HAImplTest, startService) {
+ // Valid configuration must be provided prior to starting the service.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Network state is also required.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+
+ // Start the service for DHCPv4 server.
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ // Make sure that the HA service has been created for the requested
+ // server type.
+ ASSERT_TRUE(ha_impl.service_);
+ EXPECT_EQ(HAServerType::DHCPv4, ha_impl.service_->getServerType());
+}
+
+// Tests that HAService object is created for DHCPv6 service.
+TEST_F(HAImplTest, startService6) {
+ // Valid configuration must be provided prior to starting the service.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Network state is also required.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6));
+
+ // Start the service for DHCPv4 server.
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv6));
+
+ // Make sure that the HA service has been created for the requested
+ // server type.
+ ASSERT_TRUE(ha_impl.service_);
+ EXPECT_EQ(HAServerType::DHCPv6, ha_impl.service_->getServerType());
+}
+
+// Tests for buffer4_receive callout implementation.
+TEST_F(HAImplTest, buffer4Receive) {
+ // Use hot-standby mode to make sure that this server instance is selected
+ // to process each received query. This is going to give predictable results.
+ ConstElementPtr ha_config = createValidJsonConfiguration(HAConfig::HOT_STANDBY);
+
+ // Create implementation object and configure it.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(ha_config));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ // Initially the HA service is in the waiting state and serves no scopes.
+ // We need to explicitly enable the scope to be served.
+ ha_impl.service_->serveDefaultScopes();
+
+ // Create callout handle to be used for passing arguments to the
+ // callout.
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ ASSERT_TRUE(callout_handle);
+
+ // Create the BOOTP message. We can use it for testing message parsing
+ // failure case because BOOTP is not supported. We will later turn it
+ // into the DHCP message to test successful parsing.
+ std::vector<uint8_t> msg = {
+ 1, // BOOTREQUEST
+ 1, // ethernet
+ 6, // HW address length = 6
+ 0, // hops = 0
+ 1, 2, 3, 4, // xid
+ 0, 0, // secs = 0
+ 0, 0, // flags
+ 0, 0, 0, 0, // ciaddr = 0
+ 0, 0, 0, 0, // yiaddr = 0
+ 0, 0, 0, 0, // siaddr = 0
+ 0, 0, 0, 0, // giaddr = 0
+ 1, 2, 3, 4, 5, 6, // chaddr
+ };
+
+ // fill chaddr reminder, sname and file with zeros
+ msg.insert(msg.end(), 10 + 128 + 64, 0);
+
+ // Create DHCPv4 message object from the BOOTP message. This should be
+ // successful because the message is not parsed yet.
+ Pkt4Ptr query4(new Pkt4(&msg[0], msg.size()));
+
+ // Set buffer4_receive callout arguments.
+ callout_handle->setArgument("query4", query4);
+
+ // Invoke the buffer4_receive callout.
+ ASSERT_NO_THROW(ha_impl.buffer4Receive(*callout_handle));
+
+ // The BOOTP messages are not supported so trying to unpack the message
+ // should trigger an error. The callout should set the next step to
+ // DROP treating the message as malformed.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_DROP, callout_handle->getStatus());
+ // Malformed message should not be classified.
+ EXPECT_TRUE(query4->getClasses().empty());
+
+ // Turn this into the DHCP message by appending a magic cookie and the
+ // options.
+ std::vector<uint8_t> magic_cookie = {
+ 99, 130, 83, 99
+ };
+
+ // Provide DHCP message type option, truncated vendor option and domain name.
+ // Parsing this message should be successful but vendor option should be
+ // skipped.
+ std::vector<uint8_t> options = {
+ 53, 1, 1, // Message type = DHCPDISCOVER
+ 125, 6, // vendor options
+ 1, 2, 3, 4, // enterprise id
+ 8, 1, // data len 8 but the actual length is 1 (truncated options)
+ 15, 3, 'a', 'b', 'c' // Domain name = abc
+ };
+
+ // Append the magic cookie and the options to our BOOTP message.
+ msg.insert(msg.end(), magic_cookie.begin(), magic_cookie.end());
+ msg.insert(msg.end(), options.begin(), options.end());
+
+ // Create new query and pass it to the callout.
+ query4.reset(new Pkt4(&msg[0], msg.size()));
+ callout_handle->setArgument("query4", query4);
+
+ // Invoke the callout again.
+ ASSERT_NO_THROW(ha_impl.buffer4Receive(*callout_handle));
+
+ // This time the callout should set the next step to SKIP to indicate to
+ // the DHCP server that the message has been already parsed.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_SKIP, callout_handle->getStatus());
+
+ // The client class should be assigned to the message to indicate that the
+ // server1 should process this message.
+ ASSERT_EQ(1, query4->getClasses().size());
+ EXPECT_TRUE(query4->inClass("HA_server1"));
+
+ // Check that the message has been parsed. The DHCP message type should
+ // be set in this case.
+ EXPECT_EQ(DHCPDISCOVER, static_cast<int>(query4->getType()));
+ // Vendor option should be skipped because it was truncated.
+ EXPECT_FALSE(query4->getOption(DHO_VIVSO_SUBOPTIONS));
+ // Domain name should not be skipped because the vendor option was truncated.
+ EXPECT_TRUE(query4->getOption(DHO_DOMAIN_NAME));
+}
+
+// Tests for buffer6_receive callout implementation.
+TEST_F(HAImplTest, buffer6Receive) {
+ // Use hot-standby mode to make sure that this server instance is selected
+ // to process each received query. This is going to give predictable results.
+ ConstElementPtr ha_config = createValidJsonConfiguration(HAConfig::HOT_STANDBY);
+
+ // Create implementation object and configure it.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(ha_config));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv6));
+
+ // Initially the HA service is in the waiting state and serves no scopes.
+ // We need to explicitly enable the scope to be served.
+ ha_impl.service_->serveDefaultScopes();
+
+ // Create callout handle to be used for passing arguments to the
+ // callout.
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ ASSERT_TRUE(callout_handle);
+
+ // Create DHCPv6 message. It initially has no transaction id so should be
+ // considered malformed.
+ std::vector<uint8_t> msg = {
+ 1, // Solicit
+ };
+
+ // Create DHCPv4 message object from the BOOTP message. This should be
+ // successful because the message is not parsed yet.
+ Pkt6Ptr query6(new Pkt6(&msg[0], msg.size()));
+
+ // Set buffer6_receive callout arguments.
+ callout_handle->setArgument("query6", query6);
+
+ // Invoke the buffer6_receive callout.
+ ASSERT_NO_THROW(ha_impl.buffer6Receive(*callout_handle));
+
+ // Our DHCP messages contains no transaction id so it should cause
+ // parsing error. The next step is set to DROP for malformed messages.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_DROP, callout_handle->getStatus());
+ // Malformed message should not be classified.
+ EXPECT_TRUE(query6->getClasses().empty());
+
+ // Append transaction id (3 bytes, each set to 1).
+ msg.insert(msg.end(), 3, 1);
+
+ // Include 3 options in the DHCPv6 message: ORO, truncated vendor option
+ // and the NIS Domain Name option. This should be parsed correctly but the
+ // last option should be skipped because of the preceding option being
+ // truncated.
+ std::vector<uint8_t> options = {
+ 0, 6, 0, 2, 0, 29, // option ORO requesting option 29
+ 0, 17, // vendor options
+ 0, 9, // option length = 10
+ 1, 2, 3, 4, // enterprise id
+ 0, 1, 0, 10, // code 1, (invalid) length = 10
+ 1, // ONLY 1 byte of data (truncated)
+ 0, 29, 0, 3, 'a', 'b', 'c' // NIS Domain Name = abc
+ };
+
+ msg.insert(msg.end(), options.begin(), options.end());
+
+ // Create new query and pass it to the callout.
+ query6.reset(new Pkt6(&msg[0], msg.size()));
+ callout_handle->setArgument("query6", query6);
+
+ // Invoke the callout again.
+ ASSERT_NO_THROW(ha_impl.buffer6Receive(*callout_handle));
+
+ // This time the callout should set the next step to SKIP to indicate to
+ // the DHCP server that the message has been already parsed.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_SKIP, callout_handle->getStatus());
+
+ // The client class should be assigned to the message to indicate that the
+ // server1 should process this message.
+ ASSERT_EQ(1, query6->getClasses().size());
+ EXPECT_TRUE(query6->inClass("HA_server1"));
+
+ // Check that the message has been parsed. The DHCP message type should
+ // be set in this case.
+ EXPECT_EQ(DHCPV6_SOLICIT, static_cast<int>(query6->getType()));
+ // Domain name should be skipped because the vendor option was truncated.
+ EXPECT_FALSE(query6->getOption(D6O_NIS_DOMAIN_NAME));
+}
+
+// Tests leases4_committed callout implementation.
+TEST_F(HAImplTest, leases4Committed) {
+ // Create implementation object and configure it.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ // Make sure we wait for the acks from the backup server to be able to
+ // test the case of sending lease updates even though the service is
+ // in the state in which the lease updates are normally not sent.
+ ha_impl.config_->setWaitBackupAck(true);
+
+ // Create callout handle to be used for passing arguments to the
+ // callout.
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ ASSERT_TRUE(callout_handle);
+
+ // Set the hook index so we can park packets.
+ callout_handle->setCurrentHook(test_hooks.hook_index_leases4_committed_);
+
+ // query4
+ Pkt4Ptr query4 = createMessage4(DHCPREQUEST, 1, 0, 0);
+ callout_handle->setArgument("query4", query4);
+
+ // leases4
+ Lease4CollectionPtr leases4(new Lease4Collection());
+ callout_handle->setArgument("leases4", leases4);
+
+ // deleted_leases4
+ Lease4CollectionPtr deleted_leases4(new Lease4Collection());
+ callout_handle->setArgument("deleted_leases4", deleted_leases4);
+
+ // Set initial status.
+ callout_handle->setStatus(CalloutHandle::NEXT_STEP_CONTINUE);
+
+ // Park the packet.
+ HooksManager::park("leases4_committed", query4, []{});
+
+ // There are no leases so the callout should return.
+ ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle));
+
+ // No updates are generated so the default status should not be modified.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4));
+
+ // Create a lease and pass it to the callout, but temporarily disable lease
+ // updates.
+ HWAddrPtr hwaddr(new HWAddr(std::vector<uint8_t>(6, 1), HTYPE_ETHER));
+ Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr,
+ static_cast<const uint8_t*>(0), 0,
+ 60, 0, 1));
+ leases4->push_back(lease4);
+ callout_handle->setArgument("leases4", leases4);
+
+ ha_impl.config_->setSendLeaseUpdates(false);
+
+ // Park the packet.
+ HooksManager::park("leases4_committed", query4, []{});
+
+ // Run the callout again.
+ ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle));
+
+ // No updates are generated so the default status should not be modified.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4));
+
+ // Enable updates and retry.
+ ha_impl.config_->setSendLeaseUpdates(true);
+ callout_handle->setArgument("leases4", leases4);
+
+ // Park the packet.
+ HooksManager::park("leases4_committed", query4, []{});
+
+ // Run the callout again.
+ ASSERT_NO_THROW(ha_impl.leases4Committed(*callout_handle));
+
+ // This time the lease update should be generated and the status should
+ // be set to "park".
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_PARK, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query4));
+}
+
+// Tests leases6_committed callout implementation.
+TEST_F(HAImplTest, leases6Committed) {
+ // Create implementation object and configure it.
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv6));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv6));
+
+ // Make sure we wait for the acks from the backup server to be able to
+ // test the case of sending lease updates even though the service is
+ // in the state in which the lease updates are normally not sent.
+ ha_impl.config_->setWaitBackupAck(true);
+
+ // Create callout handle to be used for passing arguments to the
+ // callout.
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ ASSERT_TRUE(callout_handle);
+
+ // Set the hook index so we can park packets.
+ callout_handle->setCurrentHook(test_hooks.hook_index_leases6_committed_);
+
+ // query6
+ Pkt6Ptr query6 = createMessage6(DHCPV6_REQUEST, 1, 0);
+ callout_handle->setArgument("query6", query6);
+
+ // leases6
+ Lease6CollectionPtr leases6(new Lease6Collection());
+ callout_handle->setArgument("leases6", leases6);
+
+ // deleted_leases6
+ Lease6CollectionPtr deleted_leases6(new Lease6Collection());
+ callout_handle->setArgument("deleted_leases6", deleted_leases6);
+
+ // Set initial status.
+ callout_handle->setStatus(CalloutHandle::NEXT_STEP_CONTINUE);
+
+ // Park the packet.
+ HooksManager::park("leases6_committed", query6, []{});
+
+ // There are no leases so the callout should return.
+ ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle));
+
+ // No updates are generated so the default status should not be modified.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6));
+
+ // Create a lease and pass it to the callout, but temporarily disable lease
+ // updates.
+ DuidPtr duid(new DUID(std::vector<uint8_t>(8, 2)));
+ Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"), duid,
+ 1234, 50, 60, 1));
+ leases6->push_back(lease6);
+ callout_handle->setArgument("leases6", leases6);
+
+ ha_impl.config_->setSendLeaseUpdates(false);
+
+ // Park the packet.
+ HooksManager::park("leases6_committed", query6, []{});
+
+ // Run the callout again.
+ ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle));
+
+ // No updates are generated so the default status should not be modified.
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_CONTINUE, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6));
+
+ // Enable updates and retry.
+ ha_impl.config_->setSendLeaseUpdates(true);
+ callout_handle->setArgument("leases6", leases6);
+
+ // Park the packet.
+ HooksManager::park("leases6_committed", query6, []{});
+
+ // Run the callout again.
+ ASSERT_NO_THROW(ha_impl.leases6Committed(*callout_handle));
+
+ // This time the lease update should be generated and the status should
+ // be set to "park".
+ EXPECT_EQ(CalloutHandle::NEXT_STEP_PARK, callout_handle->getStatus());
+ EXPECT_TRUE(callout_handle->getParkingLotHandlePtr()->drop(query6));
+}
+
+// Tests ha-sync command handler with correct and incorrect arguments.
+TEST_F(HAImplTest, synchronizeHandler) {
+ {
+ // This syntax is correct. The error returned is simply a result of
+ // trying to connect to the server which is offline, which should
+ // result in connection refused error.
+ SCOPED_TRACE("Correct syntax");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\","
+ " \"arguments\": {"
+ " \"server-name\": \"server2\""
+ " }"
+ "}", "Connection refused");
+ }
+
+ {
+ SCOPED_TRACE("No arguments");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\""
+ "}", "arguments not found in the 'ha-sync' command");
+ }
+
+ {
+ SCOPED_TRACE("No server name");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\","
+ " \"arguments\": {"
+ " \"max-period\": 20"
+ " }"
+ "}", "'server-name' is mandatory for the 'ha-sync' command");
+ }
+
+ {
+ SCOPED_TRACE("Server name is not a string");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\","
+ " \"arguments\": {"
+ " \"server-name\": 20"
+ " }"
+ "}", "'server-name' must be a string in the 'ha-sync' command");
+ }
+
+ {
+ SCOPED_TRACE("Max period is not a number");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\","
+ " \"arguments\": {"
+ " \"server-name\": \"server2\","
+ " \"max-period\": \"20\""
+ " }"
+ "}", "'max-period' must be a positive integer in the 'ha-sync'"
+ " command");
+ }
+
+ {
+ SCOPED_TRACE("Max period must be positive");
+ testSynchronizeHandler("{"
+ " \"command\": \"ha-sync\","
+ " \"arguments\": {"
+ " \"server-name\": \"server2\","
+ " \"max-period\": \"20\""
+ " }"
+ "}", "'max-period' must be a positive integer in the 'ha-sync'"
+ " command");
+ }
+
+}
+
+// Tests ha-continue command handler.
+TEST_F(HAImplTest, continueHandler) {
+ HAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ ConstElementPtr command = Element::fromJSON("{ \"command\": \"ha-continue\" }");
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ callout_handle->setArgument("command", command);
+
+ ASSERT_NO_THROW(ha_impl.continueHandler(*callout_handle));
+
+ ConstElementPtr response;
+ callout_handle->getArgument("response", response);
+ ASSERT_TRUE(response);
+
+ checkAnswer(response, CONTROL_RESULT_SUCCESS, "HA state machine is not paused.");
+}
+
+// Tests status-get command processed handler.
+TEST_F(HAImplTest, statusGet) {
+ HAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ std::string name = "status-get";
+ ConstElementPtr response =
+ Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }");
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+
+ callout_handle->setArgument("name", name);
+ callout_handle->setArgument("response", response);
+
+ ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle));
+
+ ConstElementPtr got;
+ callout_handle->getArgument("response", got);
+ ASSERT_TRUE(got);
+
+ std::string expected =
+ "{"
+ " \"arguments\": {"
+ " \"high-availability\": ["
+ " {"
+ " \"ha-mode\": \"load-balancing\","
+ " \"ha-servers\": {"
+ " \"local\": {"
+ " \"role\": \"primary\","
+ " \"scopes\": [ ],"
+ " \"state\": \"waiting\""
+ " },"
+ " \"remote\": {"
+ " \"age\": 0,"
+ " \"in-touch\": false,"
+ " \"last-scopes\": [ ],"
+ " \"last-state\": \"\","
+ " \"role\": \"secondary\","
+ " \"communication-interrupted\": false,"
+ " \"connecting-clients\": 0,"
+ " \"unacked-clients\": 0,"
+ " \"unacked-clients-left\": 0,"
+ " \"analyzed-packets\": 0"
+ " }"
+ " }"
+ " }"
+ " ],"
+ " \"pid\": 1"
+ " },"
+ " \"result\": 0"
+ "}";
+ EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected)));
+}
+
+// Tests status-get command processed handler for backup server.
+TEST_F(HAImplTest, statusGetBackupServer) {
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+ ha_impl.config_->setThisServerName("server3");
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ std::string name = "status-get";
+ ConstElementPtr response =
+ Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }");
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+
+ callout_handle->setArgument("name", name);
+ callout_handle->setArgument("response", response);
+
+ ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle));
+
+ ConstElementPtr got;
+ callout_handle->getArgument("response", got);
+ ASSERT_TRUE(got);
+
+ std::string expected =
+ "{"
+ " \"arguments\": {"
+ " \"high-availability\": ["
+ " {"
+ " \"ha-mode\": \"load-balancing\","
+ " \"ha-servers\": {"
+ " \"local\": {"
+ " \"role\": \"backup\","
+ " \"scopes\": [ ],"
+ " \"state\": \"backup\""
+ " }"
+ " }"
+ " }"
+ " ],"
+ " \"pid\": 1"
+ " },"
+ " \"result\": 0"
+ "}";
+ EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected)));
+}
+
+// Tests status-get command processed handler for primary server being in the
+// passive-backup state.
+TEST_F(HAImplTest, statusGetPassiveBackup) {
+ TestHAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidPassiveBackupJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ std::string name = "status-get";
+ ConstElementPtr response =
+ Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }");
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+
+ callout_handle->setArgument("name", name);
+ callout_handle->setArgument("response", response);
+
+ ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle));
+
+ ConstElementPtr got;
+ callout_handle->getArgument("response", got);
+ ASSERT_TRUE(got);
+
+ std::string expected =
+ "{"
+ " \"arguments\": {"
+ " \"high-availability\": ["
+ " {"
+ " \"ha-mode\": \"passive-backup\","
+ " \"ha-servers\": {"
+ " \"local\": {"
+ " \"role\": \"primary\","
+ " \"scopes\": [ \"server1\" ],"
+ " \"state\": \"passive-backup\""
+ " }"
+ " }"
+ " }"
+ " ],"
+ " \"pid\": 1"
+ " },"
+ " \"result\": 0"
+ "}";
+ EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected)));
+}
+
+// Test ha-maintenance-notify command handler.
+TEST_F(HAImplTest, maintenanceNotify) {
+ HAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ ConstElementPtr command = Element::fromJSON(
+ "{"
+ " \"command\": \"ha-maintenance-notify\","
+ " \"arguments\": {"
+ " \"cancel\": false"
+ " }"
+ "}"
+ );
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ callout_handle->setArgument("command", command);
+
+ ASSERT_NO_THROW(ha_impl.maintenanceNotifyHandler(*callout_handle));
+
+ ConstElementPtr response;
+ callout_handle->getArgument("response", response);
+ ASSERT_TRUE(response);
+
+ checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state.");
+}
+
+// Test ha-reset command handler.
+TEST_F(HAImplTest, haReset) {
+ HAImpl ha_impl;
+ ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration()));
+
+ // Starting the service is required prior to running any callouts.
+ NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4));
+ ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state,
+ HAServerType::DHCPv4));
+
+ ConstElementPtr command = Element::fromJSON(
+ "{"
+ " \"command\": \"ha-reset\""
+ "}"
+ );
+
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ callout_handle->setArgument("command", command);
+
+ ASSERT_NO_THROW(ha_impl.haResetHandler(*callout_handle));
+
+ ConstElementPtr response;
+ callout_handle->getArgument("response", response);
+ ASSERT_TRUE(response);
+
+ checkAnswer(response, CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state.");
+}
+
+}
diff --git a/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc
new file mode 100644
index 0000000..8aac8b0
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_mt_unittest.cc
@@ -0,0 +1,561 @@
+// Copyright (C) 2021-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <asiolink/asio_wrapper.h>
+#include <config/cmd_response_creator.h>
+#include <ha_test.h>
+#include <ha_config.h>
+#include <ha_service.h>
+
+#include <util/multi_threading_mgr.h>
+#include <testutils/gtest_utils.h>
+
+#include <gtest/gtest.h>
+
+using namespace isc::asiolink;
+using namespace isc::config;
+using namespace isc::data;
+using namespace isc::dhcp;
+
+using namespace isc::ha;
+using namespace isc::ha::test;
+using namespace isc::http;
+using namespace isc::util;
+
+namespace {
+
+/// @brief Derivation of the @c HAService which provides access to
+/// protected methods and members.
+class TestHAService : public HAService {
+public:
+
+ /// @brief Constructor.
+ ///
+ /// @param io_service Pointer to the IO service used by the DHCP server.
+ /// @param network_state Object holding state of the DHCP service
+ /// (enabled/disabled).
+ /// @param config Parsed HA hook library configuration.
+ /// @param server_type Server type, i.e. DHCPv4 or DHCPv6 server.
+ TestHAService(const IOServicePtr& io_service,
+ const NetworkStatePtr& network_state,
+ const HAConfigPtr& config,
+ const HAServerType& server_type = HAServerType::DHCPv4)
+ : HAService(io_service, network_state, config, server_type) {
+ }
+
+ /// @brief Test version of the @c HAService::runModel.
+ ///
+ /// The original implementation of this method returns control when
+ /// @c NOP_EVT is found. This implementation runs a
+ /// single handler to allow the tests to verify if the state machine
+ /// transitions to an expected state before it is run again.
+ virtual void runModel(unsigned int event) {
+ try {
+ postNextEvent(event);
+ getState(getCurrState())->run();
+
+ } catch (const std::exception& ex) {
+ abortModel(ex.what());
+ }
+ }
+
+ /// @brief Schedules asynchronous "dhcp-disable" command to the specified
+ /// server.
+ ///
+ /// This variant of the method uses default HTTP client for communication.
+ ///
+ /// @param server_name name of the server to which the command should be
+ /// sent.
+ /// @param max_period maximum number of seconds for which the DHCP service
+ /// should be disabled.
+ /// @param post_request_action pointer to the function to be executed when
+ /// the request is completed.
+ void asyncDisableDHCPService(const std::string& server_name,
+ const unsigned int max_period,
+ const PostRequestCallback& post_request_action) {
+ HAService::asyncDisableDHCPService(*client_, server_name, max_period,
+ post_request_action);
+ }
+
+ /// @brief Schedules asynchronous "dhcp-enable" command to the specified
+ /// server.
+ ///
+ /// This variant of the method uses default HTTP client for communication.
+ ///
+ /// @param server_name name of the server to which the command should be
+ /// sent.
+ /// @param post_request_action pointer to the function to be executed when
+ /// the request is completed.
+ void asyncEnableDHCPService(const std::string& server_name,
+ const PostRequestCallback& post_request_action) {
+ HAService::asyncEnableDHCPService(*client_, server_name, post_request_action);
+ }
+
+ using HAService::asyncSendHeartbeat;
+ using HAService::asyncSyncLeases;
+ using HAService::postNextEvent;
+ using HAService::transition;
+ using HAService::verboseTransition;
+ using HAService::shouldSendLeaseUpdates;
+ using HAService::shouldQueueLeaseUpdates;
+ using HAService::pendingRequestSize;
+ using HAService::getPendingRequest;
+ using HAService::network_state_;
+ using HAService::config_;
+ using HAService::communication_state_;
+ using HAService::query_filter_;
+ using HAService::lease_update_backlog_;
+ using HAService::client_;
+ using HAService::listener_;
+};
+
+/// @brief Pointer to the @c TestHAService.
+typedef boost::shared_ptr<TestHAService> TestHAServicePtr;
+
+/// @brief Test fixture class for @c HAService multi-threading.
+class HAMtServiceTest : public HATest {
+public:
+
+ /// @brief Constructor.
+ HAMtServiceTest()
+ : HATest() {
+ MultiThreadingMgr::instance().setMode(true);
+ CmdResponseCreator::command_accept_list_.clear();
+ }
+
+ /// @brief Destructor.
+ ///
+ /// Stops all test servers.
+ ~HAMtServiceTest() {
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+ MultiThreadingMgr::instance().setMode(false);
+ CmdResponseCreator::command_accept_list_.clear();
+ }
+
+ /// @brief Callback function invoke upon test timeout.
+ ///
+ /// It stops the IO service and reports test timeout.
+ ///
+ /// @param fail_on_timeout Specifies if test failure should be reported.
+ void timeoutHandler(const bool fail_on_timeout) {
+ if (fail_on_timeout) {
+ ADD_FAILURE() << "Timeout occurred while running the test!";
+ }
+ io_service_->stop();
+ }
+};
+
+// Verifies HA+MT start, pause, resume, and stop. Note
+// that pause and resume are tested indirectly through
+// entry and exit of a critical section.
+TEST_F(HAMtServiceTest, multiThreadingBasics) {
+
+ // Build the HA JSON configuration.
+ std::stringstream ss;
+ ss <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"wait-backup-ack\": true,"
+ " \"restrict-commands\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]";
+
+ // Enable MT, listener, and 3 threads for both client and listener.
+ ss << "," << makeHAMtJson(true, true, 3, 3) << "}]";
+ ConstElementPtr config_json;
+ ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str()));
+
+ // Enable DHCP multi-threading configuration in CfgMgr with 3 threads.
+ setDHCPMultiThreadingConfig(true, 3);
+
+ // Create the HA configuration
+ HAConfigPtr ha_config(new HAConfig());
+ HAConfigParser parser;
+ ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json));
+
+ // Instantiate the service.
+ TestHAServicePtr service;
+ ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_,
+ ha_config)));
+ // Multi-threading should be enabled.
+ ASSERT_TRUE(ha_config->getEnableMultiThreading());
+
+ // Command filtering is enabled.
+ EXPECT_FALSE(CmdResponseCreator::command_accept_list_.empty());
+
+ // Now we'll start, pause, resume and stop a few times.
+ for (int i = 0; i < 3; ++i) {
+ // Verify we're stopped.
+ // Client should exist but be stopped.
+ ASSERT_TRUE(service->client_);
+ ASSERT_TRUE(service->client_->isStopped());
+ if (i == 0) {
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+ } else {
+ EXPECT_TRUE(service->client_->getThreadIOService()->stopped());
+ }
+
+ // Listener should exist but be stopped.
+ ASSERT_TRUE(service->listener_);
+ ASSERT_TRUE(service->listener_->isStopped());
+ EXPECT_FALSE(service->listener_->getThreadIOService());
+
+ // Start client and listener.
+ ASSERT_NO_THROW_LOG(service->startClientAndListener());
+
+ // Verify we've started.
+ // Client should be running.
+ ASSERT_TRUE(service->client_->isRunning());
+ ASSERT_TRUE(service->client_->getThreadIOService());
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+ EXPECT_EQ(service->client_->getThreadPoolSize(), 3);
+ EXPECT_EQ(service->client_->getThreadCount(), 3);
+
+ // Listener should be running.
+ ASSERT_TRUE(service->listener_->isRunning());
+ ASSERT_TRUE(service->listener_->getThreadIOService());
+ EXPECT_FALSE(service->listener_->getThreadIOService()->stopped());
+ EXPECT_EQ(service->listener_->getThreadPoolSize(), 3);
+ EXPECT_EQ(service->listener_->getThreadCount(), 3);
+
+ {
+ // Entering a critical section should pause both client
+ // and listener.
+ MultiThreadingCriticalSection cs;
+
+ // Client should be paused.
+ ASSERT_TRUE(service->client_->isPaused());
+ EXPECT_TRUE(service->client_->getThreadIOService()->stopped());
+
+ // Listener should be paused.
+ ASSERT_TRUE(service->listener_->isPaused());
+ EXPECT_TRUE(service->listener_->getThreadIOService()->stopped());
+ }
+
+ // Exiting critical section should resume both client
+ // and listener.
+
+ // Client should be running.
+ ASSERT_TRUE(service->client_->isRunning());
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+
+ // Listener should be running.
+ ASSERT_TRUE(service->listener_->isRunning());
+ EXPECT_FALSE(service->listener_->getThreadIOService()->stopped());
+
+ // Stop should succeed.
+ ASSERT_NO_THROW_LOG(service->stopClientAndListener());
+ }
+}
+
+// Verifies multiThreadingBasics can be extended to use HTTPS/TLS>
+TEST_F(HAMtServiceTest, multiThreadingTls) {
+
+ // Build the HA JSON configuration.
+ std::stringstream ss;
+ ss <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"wait-backup-ack\": true,"
+ " \"require-client-certs\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"https://127.0.0.1:8080/\","
+ " \"role\": \"primary\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-server.crt\","
+ " \"key-file\": \"!CA!/kea-server.key\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"https://127.0.0.1:8081/\","
+ " \"role\": \"backup\","
+ " \"trust-anchor\": \"!CA!/kea-ca.crt\","
+ " \"cert-file\": \"!CA!/kea-client.crt\","
+ " \"key-file\": \"!CA!/kea-client.key\""
+ " }"
+ " ]";
+
+ // Enable MT, listener, and 3 threads for both client and listener.
+ ss << "," << makeHAMtJson(true, true, 3, 3) << "}]";
+ ConstElementPtr config_json;
+ const std::string& patched = replaceInConfig(ss.str(), "!CA!",
+ TEST_CA_DIR);
+ ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(patched));
+
+ // Enable DHCP multi-threading configuration in CfgMgr with 3 threads.
+ setDHCPMultiThreadingConfig(true, 3);
+
+ // Create the HA configuration
+ HAConfigPtr ha_config(new HAConfig());
+ HAConfigParser parser;
+ ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json));
+
+ // Instantiate the service.
+ TestHAServicePtr service;
+ ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_,
+ ha_config)));
+ // Multi-threading should be enabled.
+ ASSERT_TRUE(ha_config->getEnableMultiThreading());
+
+ // Now we'll start, pause, resume and stop a few times.
+ for (int i = 0; i < 3; ++i) {
+ // Verify we're stopped.
+ // Client should exist but be stopped.
+ ASSERT_TRUE(service->client_);
+ ASSERT_TRUE(service->client_->isStopped());
+ if (i == 0) {
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+ } else {
+ EXPECT_TRUE(service->client_->getThreadIOService()->stopped());
+ }
+
+ // Listener should exist but be stopped.
+ ASSERT_TRUE(service->listener_);
+ ASSERT_TRUE(service->listener_->isStopped());
+ EXPECT_FALSE(service->listener_->getThreadIOService());
+
+ // Start client and listener.
+ ASSERT_NO_THROW_LOG(service->startClientAndListener());
+
+ // Verify we've started.
+ // Client should be running.
+ ASSERT_TRUE(service->client_->isRunning());
+ ASSERT_TRUE(service->client_->getThreadIOService());
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+ EXPECT_EQ(service->client_->getThreadPoolSize(), 3);
+ EXPECT_EQ(service->client_->getThreadCount(), 3);
+
+ // Listener should be running.
+ ASSERT_TRUE(service->listener_->isRunning());
+ ASSERT_TRUE(service->listener_->getThreadIOService());
+ EXPECT_FALSE(service->listener_->getThreadIOService()->stopped());
+ EXPECT_EQ(service->listener_->getThreadPoolSize(), 3);
+ EXPECT_EQ(service->listener_->getThreadCount(), 3);
+
+ {
+ // Entering a critical section should pause both client
+ // and listener.
+ MultiThreadingCriticalSection cs;
+
+ // Client should be paused.
+ ASSERT_TRUE(service->client_->isPaused());
+ EXPECT_TRUE(service->client_->getThreadIOService()->stopped());
+
+ // Listener should be paused.
+ ASSERT_TRUE(service->listener_->isPaused());
+ EXPECT_TRUE(service->listener_->getThreadIOService()->stopped());
+ }
+
+ // Exiting critical section should resume both client
+ // and listener.
+
+ // Client should be running.
+ ASSERT_TRUE(service->client_->isRunning());
+ EXPECT_FALSE(service->client_->getThreadIOService()->stopped());
+
+ // Listener should be running.
+ ASSERT_TRUE(service->listener_->isRunning());
+ EXPECT_FALSE(service->listener_->getThreadIOService()->stopped());
+
+ // Stop should succeed.
+ ASSERT_NO_THROW_LOG(service->stopClientAndListener());
+ }
+}
+
+// Verifies permutations of HA+MT configuration and start-up.
+TEST_F(HAMtServiceTest, multiThreadingConfigStartup) {
+
+ // Structure describing a test scenario.
+ struct Scenario {
+ std::string desc_; // Description of the scenario.
+ std::string mt_json_; // multi-threading config to use.
+ bool dhcp_mt_enabled_; // True if DHCP multi-threading is enabled.
+ uint32_t dhcp_threads_; // Value of DHCP thread-pool-size.
+ bool exp_ha_mt_enabled_; // If HA+MT should be enabled
+ bool exp_listener_; // If HA+MT should use dedicated listener.
+ uint32_t exp_listener_threads_; // Expected number of listener threads.
+ uint32_t exp_client_threads_; // Expected number of client threads.
+ };
+
+ // Mnemonic constants.
+ bool dhcp_mt = true;
+ bool ha_mt = true;
+ bool listener = true;
+
+ // Number of threads the system reports as supported.
+ uint32_t sys_threads = MultiThreadingMgr::detectThreadCount();
+
+ std::vector<Scenario> scenarios {
+ {
+ "1 ha+mt by default",
+ "",
+ dhcp_mt, 4,
+ ha_mt, listener, 4, 4
+ },
+ {
+ "2 dhcp mt enabled, ha mt disabled",
+ makeHAMtJson(!ha_mt, !listener, 0, 0),
+ dhcp_mt, 4,
+ !ha_mt, !listener, 0, 0
+ },
+ {
+ "3 dhcp mt disabled, ha mt enabled",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ !dhcp_mt, 4,
+ !ha_mt, !listener, 0, 0
+ },
+ {
+ "4 dhcp mt enabled, ha mt enabled, listener disabled",
+ makeHAMtJson(ha_mt, !listener, 0, 0),
+ dhcp_mt, 4,
+ ha_mt, !listener, 4, 4
+ },
+ {
+ "5 dhcp mt enabled, ha mt enabled, listener enabled",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 4,
+ ha_mt, listener, 4, 4
+ },
+ {
+ "6 explicit DHCP threads, explicit thread values",
+ makeHAMtJson(ha_mt, listener, 5, 6),
+ dhcp_mt, 4,
+ ha_mt, listener, 5, 6
+ },
+ {
+ "7 explicit DHCP threads, zero thread values",
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 8,
+ ha_mt, listener, 8, 8
+ },
+ {
+ "8 DHCP auto detect threads, zero thread values",
+ // Special case: if system reports supported threads as 0
+ // then HA+MT should be disabled. Otherwise it should
+ // be enabled with listener and client threads set to the
+ // reported value.
+ makeHAMtJson(ha_mt, listener, 0, 0),
+ dhcp_mt, 0,
+ (sys_threads > 0), listener, sys_threads, sys_threads
+ }
+ };
+
+ // Iterate over the scenarios.
+ for (auto const& scenario : scenarios) {
+ SCOPED_TRACE(scenario.desc_);
+
+ // Build the HA JSON configuration.
+ std::stringstream ss;
+ ss <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"wait-backup-ack\": true,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:8080/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:8081/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]";
+
+ if (!scenario.mt_json_.empty()) {
+ ss << "," << scenario.mt_json_;
+ }
+
+ ss << "}]";
+ ConstElementPtr config_json;
+ ASSERT_NO_THROW_LOG(config_json = Element::fromJSON(ss.str()));
+
+ // Set DHCP multi-threading configuration in CfgMgr.
+ setDHCPMultiThreadingConfig(scenario.dhcp_mt_enabled_, scenario.dhcp_threads_);
+ /// @todo this is a hack... we have chicken-egg... CmdHttpListener won't
+ /// start if MT is not enabled BUT that happens after config hook point
+ MultiThreadingMgr::instance().setMode(scenario.dhcp_mt_enabled_);
+
+ // Create the HA configuration
+ HAConfigPtr ha_config(new HAConfig());
+ HAConfigParser parser;
+ ASSERT_NO_THROW_LOG(parser.parse(ha_config, config_json));
+
+ // Instantiate the service.
+ TestHAServicePtr service;
+ ASSERT_NO_THROW_LOG(service.reset(new TestHAService(io_service_, network_state_,
+ ha_config)));
+ ASSERT_NO_THROW_LOG(service->startClientAndListener());
+
+ // Verify the configuration is as expected.
+ if (!scenario.exp_ha_mt_enabled_) {
+ // When HA+MT is disabled, client should be single-threaded.
+ ASSERT_TRUE(service->client_);
+ EXPECT_FALSE(service->client_->getThreadIOService());
+ EXPECT_EQ(service->client_->getThreadPoolSize(), 0);
+ EXPECT_EQ(service->client_->getThreadCount(), 0);
+
+ // Listener should not exist.
+ ASSERT_FALSE(service->listener_);
+ continue;
+ }
+
+ // Multi-threading should be enabled.
+ ASSERT_TRUE(ha_config->getEnableMultiThreading());
+
+ // When HA+MT is enabled, client should be multi-threaded.
+ ASSERT_TRUE(service->client_);
+ EXPECT_TRUE(service->client_->isRunning());
+ EXPECT_TRUE(service->client_->getThreadIOService());
+ EXPECT_EQ(service->client_->getThreadPoolSize(), scenario.exp_client_threads_);
+
+ // Currently thread count should be the same as thread pool size. This might
+ // change if we go to so some sort of dynamic thread instance management.
+ EXPECT_EQ(service->client_->getThreadCount(), scenario.exp_client_threads_);
+
+ if (!scenario.exp_listener_) {
+ // We should not have a listener.
+ ASSERT_FALSE(service->listener_);
+ continue;
+ }
+
+ // We should have a running listener with the expected number of threads.
+ ASSERT_TRUE(service->listener_);
+ EXPECT_TRUE(service->listener_->isRunning());
+ ASSERT_TRUE(service->listener_->getThreadIOService());
+ EXPECT_EQ(service->listener_->getThreadPoolSize(), scenario.exp_listener_threads_);
+
+ // Currently thread count should be the same as thread pool size. This might
+ // change if we go to so some sort of dynamic thread instance management.
+ EXPECT_EQ(service->listener_->getThreadCount(), scenario.exp_listener_threads_);
+
+ ASSERT_NO_THROW_LOG(service->stopClientAndListener());
+ }
+}
+
+} // namespace
diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
new file mode 100644
index 0000000..f0c286e
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
@@ -0,0 +1,8037 @@
+// Copyright (C) 2018-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <asiolink/asio_wrapper.h>
+#include <ha_test.h>
+#include <ha_config.h>
+#include <ha_service.h>
+#include <ha_service_states.h>
+#include <asiolink/interval_timer.h>
+#include <asiolink/io_address.h>
+#include <asiolink/io_service.h>
+#include <cc/command_interpreter.h>
+#include <cc/data.h>
+#include <dhcp/classify.h>
+#include <dhcp/dhcp4.h>
+#include <dhcp/dhcp6.h>
+#include <dhcp/duid.h>
+#include <dhcp/hwaddr.h>
+#include <dhcp/pkt4.h>
+#include <dhcpsrv/cfgmgr.h>
+#include <dhcpsrv/lease.h>
+#include <dhcpsrv/lease_mgr.h>
+#include <dhcpsrv/lease_mgr_factory.h>
+#include <dhcpsrv/network_state.h>
+#include <dhcpsrv/subnet_id.h>
+#include <hooks/parking_lots.h>
+#include <http/basic_auth_config.h>
+#include <http/date_time.h>
+#include <http/http_types.h>
+#include <http/listener.h>
+#include <http/post_request_json.h>
+#include <http/response_creator.h>
+#include <http/response_creator_factory.h>
+#include <http/response_json.h>
+#include <util/multi_threading_mgr.h>
+#include <testutils/gtest_utils.h>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/pointer_cast.hpp>
+#include <boost/shared_ptr.hpp>
+#include <gtest/gtest.h>
+
+#include <functional>
+#include <sstream>
+#include <set>
+#include <string>
+#include <vector>
+
+using namespace isc::asiolink;
+using namespace isc::config;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::ha;
+using namespace isc::ha::test;
+using namespace isc::hooks;
+using namespace isc::http;
+using namespace isc::util;
+
+/// @file The tests herein were created prior to HA+MT but are very valuable
+/// for testing HA single-threaded operation and overall HA behavior.
+/// HA+MT testing is done elsewhere.
+
+namespace {
+
+/// @brief IP address to which HTTP service is bound.
+const std::string SERVER_ADDRESS = "127.0.0.1";
+
+/// @brief Port number to which HTTP service is bound.
+const unsigned short SERVER_PORT = 18123;
+
+/// @brief Request Timeout used in most of the tests (ms).
+const long REQUEST_TIMEOUT = 10000;
+
+/// @brief Persistent connection idle timeout used in most of the tests (ms).
+const long IDLE_TIMEOUT = 10000;
+
+/// @brief Test timeout (ms).
+const long TEST_TIMEOUT = 10000;
+
+/// @brief Generates IPv4 leases to be used by the tests.
+///
+/// @param [out] leases reference to the container where leases are stored.
+void generateTestLeases(std::vector<Lease4Ptr>& leases) {
+ for (uint8_t i = 1; i <= 10; ++i) {
+ uint32_t lease_address = 0xC0000201 + 256 * i;
+ std::vector<uint8_t> hwaddr(6, i);
+ Lease4Ptr lease(new Lease4(IOAddress(lease_address),
+ HWAddrPtr(new HWAddr(hwaddr, HTYPE_ETHER)),
+ ClientIdPtr(),
+ 60,
+ static_cast<time_t>(1000 + i),
+ SubnetID(i)));
+ leases.push_back(lease);
+ }
+}
+
+/// @brief Generates IPv6 leases to be used by the tests.
+///
+/// @param [out] leases reference to the container where leases are stored.
+void generateTestLeases(std::vector<Lease6Ptr>& leases) {
+ std::vector<uint8_t> address_bytes = IOAddress("2001:db8:1::1").toBytes();
+ for (uint8_t i = 1; i <= 10; ++i) {
+ DuidPtr duid(new DUID(std::vector<uint8_t>(10, i)));
+ address_bytes[6] += i;
+ Lease6Ptr lease(new Lease6(Lease::TYPE_NA,
+ IOAddress::fromBytes(AF_INET6, &address_bytes[0]),
+ duid, 1, 50, 60, SubnetID(i)));
+ leases.push_back(lease);
+ }
+}
+
+/// @brief Returns generated leases in JSON format.
+///
+/// @tparam LeasesVec vector of IPv4 or IPv6 lease pointers.
+/// @param leases reference to the container holding leases to be
+/// converted to JSON format.
+template<typename LeasesVec>
+ConstElementPtr getLeasesAsJson(const LeasesVec& leases) {
+ ElementPtr leases_json = Element::createList();
+ for (auto l = leases.begin(); l != leases.end(); ++l) {
+ leases_json->add((*l)->toElement());
+ }
+ return (leases_json);
+}
+
+/// @brief Derivation of the @c HAService which provides access to
+/// protected methods and members.
+class TestHAService : public HAService {
+public:
+
+ /// @brief Constructor.
+ ///
+ /// @param io_service Pointer to the IO service used by the DHCP server.
+ /// @param network_state Object holding state of the DHCP service
+ /// (enabled/disabled).
+ /// @param config Parsed HA hook library configuration.
+ /// @param server_type Server type, i.e. DHCPv4 or DHCPv6 server.
+ TestHAService(const IOServicePtr& io_service,
+ const NetworkStatePtr& network_state,
+ const HAConfigPtr& config,
+ const HAServerType& server_type = HAServerType::DHCPv4)
+ : HAService(io_service, network_state, config, server_type) {
+ }
+
+ /// @brief Test version of the @c HAService::runModel.
+ ///
+ /// The original implementation of this method returns control when
+ /// @c NOP_EVT is found. This implementation runs a
+ /// single handler to allow the tests to verify if the state machine
+ /// transitions to an expected state before it is run again.
+ virtual void runModel(unsigned int event) {
+ try {
+ postNextEvent(event);
+ getState(getCurrState())->run();
+
+ } catch (const std::exception& ex) {
+ abortModel(ex.what());
+ }
+ }
+
+ /// @brief Schedules asynchronous "dhcp-disable" command to the specified
+ /// server.
+ ///
+ /// This variant of the method uses default HTTP client for communication.
+ ///
+ /// @param server_name name of the server to which the command should be
+ /// sent.
+ /// @param max_period maximum number of seconds for which the DHCP service
+ /// should be disabled.
+ /// @param post_request_action pointer to the function to be executed when
+ /// the request is completed.
+ void asyncDisableDHCPService(const std::string& server_name,
+ const unsigned int max_period,
+ const PostRequestCallback& post_request_action) {
+ HAService::asyncDisableDHCPService(*client_, server_name, max_period,
+ post_request_action);
+ }
+
+ /// @brief Schedules asynchronous "dhcp-enable" command to the specified
+ /// server.
+ ///
+ /// This variant of the method uses default HTTP client for communication.
+ ///
+ /// @param server_name name of the server to which the command should be
+ /// sent.
+ /// @param post_request_action pointer to the function to be executed when
+ /// the request is completed.
+ void asyncEnableDHCPService(const std::string& server_name,
+ const PostRequestCallback& post_request_action) {
+ HAService::asyncEnableDHCPService(*client_, server_name, post_request_action);
+ }
+
+ using HAService::asyncSendHeartbeat;
+ using HAService::asyncSyncLeases;
+ using HAService::postNextEvent;
+ using HAService::transition;
+ using HAService::verboseTransition;
+ using HAService::shouldSendLeaseUpdates;
+ using HAService::shouldQueueLeaseUpdates;
+ using HAService::pendingRequestSize;
+ using HAService::getPendingRequest;
+ using HAService::network_state_;
+ using HAService::config_;
+ using HAService::communication_state_;
+ using HAService::query_filter_;
+ using HAService::lease_update_backlog_;
+ using HAService::client_;
+ using HAService::listener_;
+};
+
+/// @brief Pointer to the @c TestHAService.
+typedef boost::shared_ptr<TestHAService> TestHAServicePtr;
+
+/// @brief Test HTTP response creator.
+///
+/// It records received requests and allows the tests to retrieve them
+/// to verify that they include expected values.
+class TestHttpResponseCreator : public HttpResponseCreator {
+public:
+
+ /// @brief Constructor.
+ TestHttpResponseCreator() :
+ requests_(), control_result_(CONTROL_RESULT_SUCCESS),
+ arguments_(), per_request_control_result_(),
+ per_request_arguments_(), request_index_(), basic_auth_() {
+ }
+
+ /// @brief Removes all received requests.
+ void clearReceivedRequests() {
+ std::lock_guard<std::mutex> lk(mutex_);
+ requests_.clear();
+ }
+
+ /// @brief Returns a vector of received requests.
+ std::vector<PostHttpRequestJsonPtr> getReceivedRequests() {
+ std::lock_guard<std::mutex> lk(mutex_);
+ return (requests_);
+ }
+
+ /// @brief Finds a received request which includes two strings.
+ ///
+ /// @param str1 First string which must be included in the request.
+ /// @param str2 Second string which must be included in the request.
+ /// @param str3 Third string which must be included in the request.
+ /// It is optional and defaults to empty string which means "do not
+ /// match".
+ ///
+ /// @return Pointer to the request found, or null pointer if there is
+ /// no such request.
+ PostHttpRequestJsonPtr
+ findRequest(const std::string& str1, const std::string& str2,
+ const std::string& str3 = "") {
+ std::lock_guard<std::mutex> lk(mutex_);
+ for (auto r = requests_.begin(); r < requests_.end(); ++r) {
+ std::string request_as_string = (*r)->toString();
+ if (request_as_string.find(str1) != std::string::npos) {
+ if (request_as_string.find(str2) != std::string::npos) {
+ if (str3.empty() ||
+ (request_as_string.find(str3) != std::string::npos))
+ return (*r);
+ }
+ }
+ }
+
+ // Request not found.
+ return (PostHttpRequestJsonPtr());
+ }
+
+ /// @brief Sets control result to be included in the responses.
+ ///
+ /// @param control_result new control result value.
+ void setControlResult(const int control_result) {
+ control_result_ = control_result;
+ }
+
+ /// @brief Sets control result to be returned for the particular command.
+ ///
+ /// @param command_name command name.
+ /// @param control_result new control result value.
+ void setControlResult(const std::string& command_name,
+ const int control_result) {
+ std::lock_guard<std::mutex> lk(mutex_);
+ per_request_control_result_[command_name] = control_result;
+ }
+
+ /// @brief Sets arguments to be included in the responses.
+ ///
+ /// @param arguments pointer to the arguments.
+ void setArguments(const ElementPtr& arguments) {
+ std::lock_guard<std::mutex> lk(mutex_);
+ arguments_ = arguments;
+ }
+
+ /// @brief Sets arguments to be included in the response to a particular
+ /// command.
+ ///
+ /// Some tests require that the returned arguments vary for the same
+ /// command sent multiple times. One example of such command is the
+ /// @c lease4-get-page command sent multiple times to fetch pages of
+ /// leases from the partner. This method facilitates such test scenario.
+ /// In order to set different arguments for consecutive requests this
+ /// method must be called multiple times. Each call results in adding
+ /// arguments to be returned when the command is issued multiple times.
+ ///
+ /// @param command_name command name.
+ /// @param arguments pointer to the arguments.
+ void setArguments(const std::string& command_name,
+ const ElementPtr& arguments) {
+ std::lock_guard<std::mutex> lk(mutex_);
+ per_request_arguments_[command_name].push_back(isc::data::copy(arguments));
+ // Create request index for this command if it doesn't exist.
+ if (request_index_.count(command_name) == 0) {
+ request_index_[command_name] = 0;
+ }
+ }
+
+ /// @brief Create a new request.
+ ///
+ /// @return Pointer to the new instance of the @ref HttpRequest.
+ virtual HttpRequestPtr
+ createNewHttpRequest() const {
+ return (HttpRequestPtr(new PostHttpRequestJson()));
+ }
+
+ /// @brief Get basic HTTP authentication credentials.
+ ///
+ /// @return Basic HTTP authentication credentials and user id map.
+ const BasicHttpAuthMap& getCredentials() const {
+ return (basic_auth_.getCredentialMap());
+ }
+
+ /// @brief Add basic HTTP authentication client.
+ ///
+ /// @param user The user id to authorize.
+ /// @param password The password.
+ void addBasicAuth(const std::string& user, const std::string& password) {
+ std::lock_guard<std::mutex> lk(mutex_);
+ basic_auth_.add(user, "", password, "");
+ }
+
+private:
+
+ /// @brief Creates HTTP response.
+ ///
+ /// @param request Pointer to the HTTP request.
+ /// @return Pointer to the generated HTTP response.
+ virtual HttpResponsePtr
+ createStockHttpResponse(const HttpRequestPtr& request,
+ const HttpStatusCode& status_code) const {
+ // The request hasn't been finalized so the request object
+ // doesn't contain any information about the HTTP version number
+ // used. But, the context should have this data (assuming the
+ // HTTP version is parsed ok).
+ HttpVersion http_version(request->context()->http_version_major_,
+ request->context()->http_version_minor_);
+ // This will generate the response holding JSON content.
+ HttpResponseJsonPtr response(new HttpResponseJson(http_version, status_code));
+ response->finalize();
+ return (response);
+ }
+
+ /// @brief Creates HTTP response.
+ ///
+ /// It records received request so it may be later validated by the tests.
+ /// The returned status code and arguments are set using @c setControlResult
+ /// and @c setArguments methods. The per-request control result and arguments
+ /// take precedence over global values.
+ ///
+ /// @param request Pointer to the HTTP request.
+ /// @return Pointer to the generated HTTP OK response.
+ virtual HttpResponsePtr
+ createDynamicHttpResponse(HttpRequestPtr request) {
+ std::unique_lock<std::mutex> lk(mutex_);
+ // Check authentication.
+ const BasicHttpAuthMap& credentials = getCredentials();
+ if (!credentials.empty()) {
+ bool authorized = false;
+ try {
+ std::string value = request->getHeaderValue("Authorization");
+ if (value.size() < 8) {
+ isc_throw(isc::BadValue, "header content is too short");
+ }
+ if (value.substr(0, 6) != "Basic ") {
+ isc_throw(isc::BadValue, "no basic authentication");
+ }
+ value = value.substr(6);
+ if (credentials.count(value) != 0) {
+ authorized = true;
+ }
+ } catch (const std::exception&) {
+ authorized = false;
+ }
+ if (!authorized) {
+ return (createStockHttpResponse(request,
+ HttpStatusCode::UNAUTHORIZED));
+ }
+ }
+
+ // Request must always be JSON.
+ PostHttpRequestJsonPtr request_json =
+ boost::dynamic_pointer_cast<PostHttpRequestJson>(request);
+
+ // Remember the request received.
+ requests_.push_back(request_json);
+
+ // The request must always contain non-empty Host header.
+ bool invalid_host = false;
+ try {
+ auto host_hdr = request_json->getHeader("Host");
+ if (host_hdr->getValue().empty()) {
+ invalid_host = true;
+ }
+
+ } catch (...) {
+ // Host header does not exist.
+ invalid_host = true;
+ }
+
+ // If invalid host then return Bad Request.
+ if (invalid_host) {
+ return (createStockHttpResponse(request, HttpStatusCode::BAD_REQUEST));
+ }
+
+ int control_result = -1;
+ ElementPtr arguments;
+
+ // First, check if the request contains a body with a command.
+ // If so, we may include a specific error code and arguments in the
+ // response based on the command name.
+ ConstElementPtr body = request_json->getBodyAsJson();
+ if (body && (body->getType() == Element::map)) {
+ ConstElementPtr command = body->get("command");
+ if (command && (command->getType() == Element::string)) {
+ std::string command_name = command->stringValue();
+
+ // Check if there is specific error code to be returned for this
+ // command.
+ if (per_request_control_result_.count(command_name) > 0) {
+ control_result = per_request_control_result_[command_name];
+ }
+
+ // Check if there are specific arguments to be returned for this
+ // command.
+ if (per_request_arguments_.count(command_name) > 0) {
+ // For certain requests we may return different arguments for consecutive
+ // instances of the same command. The request_index_ tracks the current
+ // index of the arguments to be returned.
+ arguments = per_request_arguments_[command_name][request_index_[command_name]];
+
+ // If we have reached the last arguments do not increase the index. Simply
+ // continue returning the same arguments.
+ if (request_index_[command_name] + 1 < per_request_arguments_[command_name].size()) {
+ // Not the last arguments, increase the index.
+ ++request_index_[command_name];
+ }
+ }
+ }
+ }
+
+ HttpResponseJsonPtr response(new HttpResponseJson(request->getHttpVersion(),
+ HttpStatusCode::OK));
+ // Body is a list of responses from multiple servers listed in "service"
+ // argument of the request.
+ ElementPtr response_body = Element::createList();
+
+ // No per-command control result specified, so include the global result.
+ if (control_result < 0) {
+ control_result = control_result_;
+ }
+
+ // No per-command arguments specified, so include the global arguments.
+ if (!arguments) {
+ arguments = arguments_;
+ }
+
+ // Insert current date-time if not statically provided.
+ if (arguments && !arguments->contains("date-time")) {
+ arguments->set("date-time", Element::create(HttpDateTime().rfc1123Format()));
+ }
+
+ // Insert unsent-update-count if not present.
+ if (arguments && !arguments->contains("unsent-update-count")) {
+ arguments->set("unsent-update-count", Element::create(int64_t(0)));
+ }
+
+ response_body->add(boost::const_pointer_cast<Element>
+ (createAnswer(control_result, "response returned",
+ arguments)));
+ response->setBodyAsJson(response_body);
+ response->finalize();
+ return (response);
+ }
+
+ /// @brief Holds received HTTP requests.
+ std::vector<PostHttpRequestJsonPtr> requests_;
+
+ /// @brief Control result to be returned in the server responses.
+ int control_result_;
+
+ /// @brief Arguments to be included in the responses.
+ ElementPtr arguments_;
+
+ /// @brief Command specific control results.
+ std::map<std::string, int> per_request_control_result_;
+
+ /// @brief Command specific response arguments.
+ std::map<std::string, std::vector<ElementPtr> > per_request_arguments_;
+
+ /// @brief Index of the next request of the given type.
+ std::map<std::string, size_t> request_index_;
+
+ /// @brief Basic HTTP authentication configuration.
+ BasicHttpAuthConfig basic_auth_;
+
+ /// @brief Mutex to protect member access.
+ std::mutex mutex_;
+};
+
+/// @brief Shared pointer to the @c TestHttpResponseCreator.
+typedef boost::shared_ptr<TestHttpResponseCreator> TestHttpResponseCreatorPtr;
+
+/// @brief Implementation of the test @ref HttpResponseCreatorFactory.
+///
+/// This factory class creates @ref TestHttpResponseCreator instances.
+class TestHttpResponseCreatorFactory : public HttpResponseCreatorFactory {
+public:
+
+ /// @brief Constructor.
+ ///
+ /// Initializes common HTTP response creator instance.
+ TestHttpResponseCreatorFactory()
+ : creator_(new TestHttpResponseCreator()) {
+ }
+
+ /// @brief Creates @ref TestHttpResponseCreator instance.
+ virtual HttpResponseCreatorPtr create() const {
+ return (creator_);
+ }
+
+ /// @brief Returns instance of the response creator constructed by this
+ /// factory.
+ TestHttpResponseCreatorPtr getResponseCreator() const {
+ return (boost::dynamic_pointer_cast<TestHttpResponseCreator>(creator_));
+ }
+
+private:
+
+ /// @brief Pointer to the common HTTP response creator.
+ HttpResponseCreatorPtr creator_;
+};
+
+/// @brief Pointer to the @c TestHttpResponseCreatorFactory.
+typedef boost::shared_ptr<TestHttpResponseCreatorFactory>
+TestHttpResponseCreatorFactoryPtr;
+
+/// @brief Test fixture class for @c HAService.
+///
+/// It creates 3 HTTP listeners (servers) which are used in the unit tests.
+class HAServiceTest : public HATest {
+public:
+
+ struct MyState {
+ explicit MyState(const int state)
+ : state_(state) {
+ }
+ int state_;
+ };
+
+ struct PartnerState {
+ explicit PartnerState(const int state)
+ : state_(state) {
+ }
+ int state_;
+ };
+
+ struct FinalState {
+ explicit FinalState(const int state)
+ : state_(state) {
+ }
+ int state_;
+ };
+
+ /// @brief Constructor.
+ HAServiceTest()
+ : HATest(),
+ service_(),
+ factory_(new TestHttpResponseCreatorFactory()),
+ factory2_(new TestHttpResponseCreatorFactory()),
+ factory3_(new TestHttpResponseCreatorFactory()),
+ listener_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS),
+ SERVER_PORT, TlsContextPtr(), factory_,
+ HttpListener::RequestTimeout(REQUEST_TIMEOUT),
+ HttpListener::IdleTimeout(IDLE_TIMEOUT))),
+ listener2_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS),
+ SERVER_PORT + 1, TlsContextPtr(), factory2_,
+ HttpListener::RequestTimeout(REQUEST_TIMEOUT),
+ HttpListener::IdleTimeout(IDLE_TIMEOUT))),
+ listener3_(new HttpListener(*io_service_, IOAddress(SERVER_ADDRESS),
+ SERVER_PORT + 2, TlsContextPtr(), factory3_,
+ HttpListener::RequestTimeout(REQUEST_TIMEOUT),
+ HttpListener::IdleTimeout(IDLE_TIMEOUT))),
+ leases4_(),
+ leases6_(),
+ user1_(""),
+ password1_(""),
+ user2_(""),
+ password2_(""),
+ user3_(""),
+ password3_("") {
+ MultiThreadingMgr::instance().setMode(false);
+ }
+
+ /// @brief Destructor.
+ ///
+ /// Stops all test servers.
+ ~HAServiceTest() {
+ listener_->stop();
+ listener2_->stop();
+ listener3_->stop();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+ MultiThreadingMgr::instance().setMode(false);
+ }
+
+ /// @brief Callback function invoke upon test timeout.
+ ///
+ /// It stops the IO service and reports test timeout.
+ ///
+ /// @param fail_on_timeout Specifies if test failure should be reported.
+ void timeoutHandler(const bool fail_on_timeout) {
+ if (fail_on_timeout) {
+ ADD_FAILURE() << "Timeout occurred while running the test!";
+ }
+ io_service_->stop();
+ }
+
+ /// @brief Creates a TestHAService instance with HA+MT disabled.
+ ///
+ /// @param network_state Object holding state of the DHCP service
+ /// (enabled/disabled).
+ /// @param config Parsed HA hook library configuration.
+ /// @param server_type server type, i.e. DHCPv4 or DHCPv6.
+ void createSTService(const NetworkStatePtr& state,
+ const HAConfigPtr config,
+ const HAServerType& server_type = HAServerType::DHCPv4) {
+
+ ASSERT_FALSE(config->getEnableMultiThreading());
+ ASSERT_NO_THROW_LOG(service_.reset(new TestHAService(io_service_, state,
+ config, server_type)));
+ ASSERT_TRUE(service_->client_);
+ ASSERT_FALSE(service_->client_->getThreadIOService());
+ ASSERT_FALSE(service_->listener_);
+ }
+
+ /// @brief Generates IPv4 leases to be used by the tests.
+ void generateTestLeases4() {
+ generateTestLeases(leases4_);
+ }
+
+ /// @brief Returns range of generated IPv4 leases in JSON format.
+ ///
+ /// @param first_index Index of the first lease to be returned.
+ /// @param last_index Index of the last lease to be returned.
+ ConstElementPtr getTestLeases4AsJson(const size_t first_index,
+ const size_t last_index) const {
+ return (getLeasesAsJson(std::vector<Lease4Ptr>(leases4_.begin() + first_index,
+ leases4_.begin() + last_index)));
+ }
+
+ /// @brief Generates IPv6 leases to be used by the tests.
+ void generateTestLeases6() {
+ generateTestLeases(leases6_);
+ }
+
+ /// @brief Returns range of generated IPv6 leases in JSON format.
+ ///
+ /// @param first_index Index of the first lease to be returned.
+ /// @param last_index Index of the last lease to be returned.
+ ConstElementPtr getTestLeases6AsJson(const size_t first_index,
+ const size_t last_index) const {
+ return (getLeasesAsJson(std::vector<Lease6Ptr>(leases6_.begin() + first_index,
+ leases6_.begin() + last_index)));
+ }
+
+ /// @brief Configure server side to return IPv4 leases in 3-element
+ /// chunks.
+ ///
+ /// Leases are fetched in pages, so the lease4-get-page should be
+ /// sent multiple times. The server is configured to return leases
+ /// in 3-element chunks. Note that the HA configis set to ask for 3
+ /// leases.
+ void createPagedSyncResponses4() {
+ ElementPtr response_arguments = Element::createMap();
+
+ // First, return leases with indexes from 0 to 2.
+ response_arguments->set("leases", getTestLeases4AsJson(0, 3));
+ factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+
+ // Next, return leases with indexes from 3 to 5.
+ response_arguments->set("leases", getTestLeases4AsJson(3, 6));
+ factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+
+ // Then, return leases with indexes from 6 to 8.
+ response_arguments->set("leases", getTestLeases4AsJson(6, 9));
+ factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+
+ // Finally, there is one lease with index 9 to be returned.
+ // When the server requests a page of 3 leases and gets 1 it
+ // means that the last page was returned. At this point, the
+ // server ends synchronization.
+ response_arguments->set("leases", getTestLeases4AsJson(9, 10));
+ factory2_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ }
+
+ /// @brief Configure server side to return IPv6 leases in 3-element
+ /// chunks.
+ ///
+ /// Leases are fetched in pages, so the lease6-get-page should be
+ /// sent multiple times. The server is configured to return leases
+ /// in 3-element chunks. Note that the HA configis set to ask for 3
+ /// leases.
+ void createPagedSyncResponses6() {
+ ElementPtr response_arguments = Element::createMap();
+
+ // First, return leases with indexes from 0 to 2.
+ response_arguments->set("leases", getTestLeases6AsJson(0, 3));
+ factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+
+ // Next, return leases with indexes from 3 to 5.
+ response_arguments->set("leases", getTestLeases6AsJson(3, 6));
+ factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+
+ // Then, return leases with indexes from 6 to 8.
+ response_arguments->set("leases", getTestLeases6AsJson(6, 9));
+ factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+
+ // Finally, there is one lease with index 9 to be returned.
+ // When the server requests a page of 3 leases and gets 1 it
+ // means that the last page was returned. At this point, the
+ // server ends synchronization.
+ response_arguments->set("leases", getTestLeases6AsJson(9, 10));
+ factory2_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+ factory3_->getResponseCreator()->setArguments("lease6-get-page", response_arguments);
+ }
+
+ /// @brief Set basic HTTP authentication in a config.
+ ///
+ /// @param config Configuration to update.
+ void setBasicAuth(HAConfigPtr config) {
+ if (!config) {
+ ADD_FAILURE() << "null config";
+ return;
+ }
+ if (!user1_.empty()) {
+ HAConfig::PeerConfigPtr peer = config->getPeerConfig("server1");
+ if (!peer) {
+ ADD_FAILURE() << "null server1 config";
+ return;
+ }
+ BasicHttpAuthPtr& auth = peer->getBasicAuth();
+ auth.reset(new BasicHttpAuth(user1_, password1_));
+ }
+ if (!user2_.empty()) {
+ HAConfig::PeerConfigPtr peer = config->getPeerConfig("server2");
+ if (!peer) {
+ ADD_FAILURE() << "null server2 config";
+ return;
+ }
+ BasicHttpAuthPtr& auth = peer->getBasicAuth();
+ auth.reset(new BasicHttpAuth(user2_, password2_));
+ }
+ if (!user3_.empty()) {
+ HAConfig::PeerConfigPtr peer = config->getPeerConfig("server3");
+ if (!peer) {
+ ADD_FAILURE() << "null server3 config";
+ return;
+ }
+ BasicHttpAuthPtr& auth = peer->getBasicAuth();
+ auth.reset(new BasicHttpAuth(user3_, password3_));
+ }
+ }
+
+ /// @brief Tests scenarios when lease updates are sent to a partner while
+ /// the partner is online or offline.
+ ///
+ /// @param unpark_handler a function called when packet is unparked.
+ /// @param should_fail indicates if the update is expected to be unsuccessful.
+ /// @param num_updates expected number of servers to which lease updates are
+ /// sent.
+ /// @param my_state state of the server while lease updates are sent.
+ /// @param wait_backup_ack indicates if the server should wait for the acknowledgment
+ /// from the backup servers.
+ /// @param create_service a boolean flag indicating whether the test should
+ /// re-create HA service and communication state.
+ void testSendLeaseUpdates(std::function<void()> unpark_handler,
+ const bool should_fail,
+ const size_t num_updates,
+ const MyState& my_state = MyState(HA_LOAD_BALANCING_ST),
+ const bool wait_backup_ack = false,
+ const bool create_service = true) {
+ // Create parking lot where query is going to be parked and unparked.
+ ParkingLotPtr parking_lot(new ParkingLot());
+ ParkingLotHandlePtr parking_lot_handle(new ParkingLotHandle(parking_lot));
+
+ // Create query.
+ Pkt4Ptr query(new Pkt4(DHCPREQUEST, 1234));
+
+ // Create leases collection and put the lease there.
+ Lease4CollectionPtr leases4(new Lease4Collection());
+ HWAddrPtr hwaddr(new HWAddr(std::vector<uint8_t>(6, 1), HTYPE_ETHER));
+ Lease4Ptr lease4(new Lease4(IOAddress("192.1.2.3"), hwaddr,
+ static_cast<const uint8_t*>(0), 0,
+ 60, 0, 1));
+ leases4->push_back(lease4);
+
+ // Create deleted leases collection and put the lease there too.
+ Lease4CollectionPtr deleted_leases4(new Lease4Collection());
+ Lease4Ptr deleted_lease4(new Lease4(IOAddress("192.2.3.4"), hwaddr,
+ static_cast<const uint8_t*>(0), 0,
+ 60, 0, 1));
+ deleted_leases4->push_back(deleted_lease4);
+
+ if (create_service) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ config_storage->setWaitBackupAck(wait_backup_ack);
+ // Let's override the default value. The lower value makes it easier
+ // for some unit tests to simulate the server's overflow. Simulating it
+ // requires appending leases to the backlog. It is easier to add 10
+ // than 100.
+ config_storage->setDelayedUpdatesLimit(10);
+ setBasicAuth(config_storage);
+
+ // The communication state is the member of the HAServce object. We have to
+ // replace this object with our own implementation to have an ability to
+ // modify its poke time.
+ NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_,
+ config_storage));
+ // Set poke time 30s in the past. If the state is poked it will be reset
+ // to the current time. This allows for testing whether the object has been
+ // poked by the HA service.
+ state->modifyPokeTime(-30);
+
+ // Create HA service.
+ createSTService(network_state_, config_storage);
+ service_->communication_state_ = state;
+ }
+
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+
+ // Schedule lease updates.
+ EXPECT_EQ(num_updates,
+ service_->asyncSendLeaseUpdates(query, leases4, deleted_leases4,
+ parking_lot_handle));
+
+ // The number of pending requests should be 2 times the number of
+ // contacted servers because we send one lease update and one
+ // lease deletion to each contacted server from which we expect
+ // an acknowledgment.
+ EXPECT_EQ(2 * num_updates, service_->getPendingRequest(query));
+
+ // Let's park the packet and associate it with the callback function which
+ // simply records the fact that it has been called. We expect that it wasn't
+ // because the parked packet should be dropped as a result of lease updates
+ // failures.
+ ASSERT_NO_THROW(parking_lot->park(query, unpark_handler));
+
+ ASSERT_NO_THROW(parking_lot->reference(query));
+
+ // Actually perform the lease updates.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, [this]() {
+ // Finish running IO service when there are no more pending requests.
+ return (service_->pendingRequestSize() == 0);
+ }));
+
+ // Only if we wait for lease updates to complete it makes sense to test
+ // that the packet was either dropped or unparked.
+ if (num_updates > 0) {
+ // Try to drop the packet. We expect that the packet has been already
+ // dropped so this should return false.
+ EXPECT_FALSE(parking_lot_handle->drop(query));
+ }
+
+ // The updates should not be sent to this server.
+ EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty());
+
+ if (should_fail) {
+ EXPECT_EQ(HA_UNAVAILABLE_ST, service_->communication_state_->getPartnerState());
+ } else {
+ EXPECT_NE(service_->communication_state_->getPartnerState(), HA_UNAVAILABLE_ST);
+ }
+ }
+
+ /// @brief Tests scenarios when IPv6 lease updates are sent to a partner while
+ /// the partner is online or offline.
+ ///
+ /// @param unpark_handler a function called when packet is unparked.
+ /// @param should_fail indicates if the update is expected to be unsuccessful.
+ /// @param num_updates expected number of servers to which lease updates are
+ /// sent.
+ /// @param my_state state of the server while lease updates are sent.
+ /// @param wait_backup_ack indicates if the server should wait for the acknowledgment
+ /// from the backup servers.
+ /// @param create_service a boolean flag indicating whether the test should
+ /// re-create HA service and communication state.
+ void testSendLeaseUpdates6(std::function<void()> unpark_handler,
+ const bool should_fail,
+ const size_t num_updates,
+ const MyState& my_state = MyState(HA_LOAD_BALANCING_ST),
+ const bool wait_backup_ack = false,
+ const bool create_service = true) {
+
+ // Create parking lot where query is going to be parked and unparked.
+ ParkingLotPtr parking_lot(new ParkingLot());
+ ParkingLotHandlePtr parking_lot_handle(new ParkingLotHandle(parking_lot));
+
+ DuidPtr duid(new DUID(std::vector<uint8_t>(8, 2)));
+
+ // Create query.
+ Pkt6Ptr query(new Pkt6(DHCPV6_SOLICIT, 1234));
+ OptionPtr opt_client_id(new Option(Option::V6,
+ D6O_CLIENTID,
+ duid->getDuid().begin(),
+ duid->getDuid().end()));
+ query->addOption(opt_client_id);
+
+ // Create leases collection and put the lease there.
+ Lease6CollectionPtr leases6(new Lease6Collection());
+ Lease6Ptr lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::cafe"), duid,
+ 1234, 50, 60, 1));
+ leases6->push_back(lease6);
+
+ // Create deleted leases collection and put the lease there too.
+ Lease6CollectionPtr deleted_leases6(new Lease6Collection());
+ Lease6Ptr deleted_lease6(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::efac"),
+ duid, 1234, 50, 60, 1));
+ deleted_leases6->push_back(deleted_lease6);
+
+ if (create_service) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ config_storage->setDelayedUpdatesLimit(10);
+ config_storage->setWaitBackupAck(wait_backup_ack);
+ setBasicAuth(config_storage);
+
+ // The communication state is the member of the HAServce object. We have to
+ // replace this object with our own implementation to have an ability to
+ // modify its poke time.
+ NakedCommunicationState6Ptr state(new NakedCommunicationState6(io_service_,
+ config_storage));
+ // Set poke time 30s in the past. If the state is poked it will be reset
+ // to the current time. This allows for testing whether the object has been
+ // poked by the HA service.
+ state->modifyPokeTime(-30);
+
+ // Create HA service.
+ createSTService(network_state_, config_storage, HAServerType::DHCPv6);
+ service_->communication_state_ = state;
+ }
+
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+
+ // Schedule lease updates.
+ EXPECT_EQ(num_updates,
+ service_->asyncSendLeaseUpdates(query, leases6, deleted_leases6,
+ parking_lot_handle));
+
+ // The number of requests we send is equal to the number of servers
+ // from which we expect an acknowledgement. We send both lease updates
+ // and the deletions in a single bulk update command.
+ EXPECT_EQ(num_updates, service_->getPendingRequest(query));
+
+ EXPECT_FALSE(boost::dynamic_pointer_cast<NakedCommunicationState6>
+ (service_->communication_state_)->isPoked());
+
+ // Let's park the packet and associate it with the callback function which
+ // simply records the fact that it has been called. We expect that it wasn't
+ // because the parked packet should be dropped as a result of lease updates
+ // failures.
+ ASSERT_NO_THROW(parking_lot->park(query, unpark_handler));
+
+ ASSERT_NO_THROW(parking_lot->reference(query));
+
+ // Actually perform the lease updates.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, [this]() {
+ // Finish running IO service when there are no more pending requests.
+ return (service_->pendingRequestSize() == 0);
+ }));
+
+ // Only if we wait for lease updates to complete it makes sense to test
+ // that the packet was either dropped or unparked.
+ if (num_updates > 0) {
+ // Try to drop the packet. We expect that the packet has been already
+ // dropped so this should return false.
+ EXPECT_FALSE(parking_lot_handle->drop(query));
+ }
+
+ // The updates should not be sent to this server.
+ EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty());
+
+ if (should_fail) {
+ EXPECT_EQ(HA_UNAVAILABLE_ST, service_->communication_state_->getPartnerState());
+ }
+ }
+
+ /// @brief Tests scenarios when recurring heartbeat has been enabled
+ /// and the partner is online or offline.
+ ///
+ /// @param control_result control result that the servers should return.
+ /// @param should_pass boolean value indicating if the heartbeat should
+ /// be successful or not.
+ void testRecurringHeartbeat(const int control_result,
+ const bool should_pass) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ config_storage->setHeartbeatDelay(1000);
+ setBasicAuth(config_storage);
+
+ // Create a valid static response to the heartbeat command.
+ ElementPtr response_arguments = Element::createMap();
+ response_arguments->set("state", Element::create(std::string("load-balancing")));
+
+ // Both server 2 and server 3 are configured to send this response.
+ factory2_->getResponseCreator()->setArguments(response_arguments);
+ factory3_->getResponseCreator()->setArguments(response_arguments);
+
+ // Configure server 2 and server 3 to send a specified control result.
+ factory2_->getResponseCreator()->setControlResult(control_result);
+ factory3_->getResponseCreator()->setControlResult(control_result);
+
+ // The communication state is the member of the HAServce object. We have to
+ // replace this object with our own implementation to have an ability to
+ // modify its poke time.
+ NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_,
+ config_storage));
+ // Set poke time 30s in the past. If the state is poked it will be reset
+ // to the current time. This allows for testing whether the object has been
+ // poked by the HA service.
+ state->modifyPokeTime(-30);
+
+ // Create the service and replace the default communication state object.
+ TestHAService service(io_service_, network_state_, config_storage);
+ service.communication_state_ = state;
+
+ EXPECT_FALSE(state->isPoked());
+
+ // Let's explicitly transition the state machine to the load balancing state
+ // in which the periodic heartbeats should be generated.
+ ASSERT_NO_THROW(service.verboseTransition(HA_LOAD_BALANCING_ST));
+ ASSERT_NO_THROW(service.runModel(HAService::NOP_EVT));
+
+ // Run the IO service to allow the heartbeat interval timers to execute.
+ ASSERT_NO_THROW(runIOService(2000));
+
+ // Server 1 and server 3 must never receive heartbeats because the former
+ // is the one that generates them and the latter is a backup server.
+ EXPECT_TRUE(factory_->getResponseCreator()->getReceivedRequests().empty());
+ EXPECT_TRUE(factory3_->getResponseCreator()->getReceivedRequests().empty());
+
+ // If should pass, the communication state should be poked.
+ if (should_pass) {
+ EXPECT_TRUE(state->isPoked());
+ } else {
+ EXPECT_FALSE(state->isPoked());
+ }
+ }
+
+ /// @brief Tests scenarios when all lease updates are sent successfully.
+ void testSendSuccessfulUpdates() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] { unpark_called = true; },
+ false, 1);
+
+ // Expecting that the packet was unparked because lease
+ // updates are expected to be successful.
+ EXPECT_TRUE(unpark_called);
+
+ // Updates have been sent so this counter should remain 0.
+ EXPECT_EQ(0, service_->communication_state_->getUnsentUpdateCount());
+
+ // The server 2 should have received two commands.
+ EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 2 has received lease4-update command.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request2);
+
+ // Check that the server 2 has received lease4-del command.
+ auto delete_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request2);
+
+ // Lease updates should be successfully sent to server3.
+ EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 3 has received lease4-update command.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request3);
+
+ // Check that the server 3 has received lease4-del command.
+ auto delete_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request3);
+ }
+
+ /// @brief Tests that DHCPv4 lease updates are queued when the server is in the
+ /// communication-recovery state and later sent before transitioning back to
+ /// the load-balancing state.
+ void testSendUpdatesCommunicationRecovery() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST));
+
+ // Packet shouldn't be unparked because no updates went out. We merely
+ // queued the updates.
+ EXPECT_FALSE(unpark_called);
+
+ // Let's make sure they have been queued.
+ EXPECT_EQ(2, service_->lease_update_backlog_.size());
+
+ // Make partner available.
+ service_->communication_state_->poke();
+ service_->communication_state_->setPartnerState("load-balancing");
+
+ // This should cause the server to send outstanding lease updates and
+ // because they are all successful the server should transition to the
+ // load-balancing state and continue normal operation.
+ testSynchronousCommands([this]() {
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+ });
+
+ // Lease updates should have been sent.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4"));
+
+ // Backlog should be empty.
+ EXPECT_EQ(0, service_->lease_update_backlog_.size());
+ }
+
+ /// @brief Test the cases when the trying to recover from the communication
+ /// interruption and sending lease updates or/and ha-reset fails.
+ ///
+ /// @param partner_state partner state when communication is re-established.
+ /// @param lease_update_result control result returned to lease updates.
+ /// @param ha_reset_result control result returned to ha-reset command.
+ /// @param overflow boolean value indicating if this test should verify the
+ /// case when the leases backlog is overflown (when true), or not (when
+ /// false).
+ void testSendUpdatesCommunicationRecoveryFailed(const std::string& partner_state,
+ const int lease_update_result,
+ const int ha_reset_result,
+ const bool overflow = false) {
+ // Partner responds with a specified control result to lease updates.
+ factory2_->getResponseCreator()->setControlResult("lease4-update",
+ lease_update_result);
+ factory2_->getResponseCreator()->setControlResult("lease4-del",
+ lease_update_result);
+ // Partner returns specified control result to ha-reset.
+ factory2_->getResponseCreator()->setControlResult("ha-reset", ha_reset_result);
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST));
+
+ // Packet shouldn't be unparked because no updates went out. We merely
+ // queued the updates.
+ EXPECT_FALSE(unpark_called);
+
+ // Let's make sure they have been queued.
+ EXPECT_EQ(2, service_->lease_update_backlog_.size());
+
+ // When testing the case when the backlog should be overflown, we need
+ // to add several more leases to the backlog to exceed the limit.
+ if (overflow) {
+ ASSERT_NO_THROW(generateTestLeases4());
+ for (auto lease : leases4_) {
+ service_->lease_update_backlog_.push(LeaseUpdateBacklog::ADD, lease);
+ }
+ }
+
+ // Make partner available.
+ service_->communication_state_->poke();
+ service_->communication_state_->setPartnerState(partner_state);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This should cause the server to attempt to send outstanding lease
+ // updates to the partner.
+ testSynchronousCommands([this, ha_reset_result]() {
+ service_->runModel(HAService::NOP_EVT);
+ // If the ha-reset returns success the server should transition to the
+ // waiting state and begin synchronization. Otherwise, if the ha-reset
+ // fails the server should wait in the communication-recovery state
+ // until it succeeds.
+ if (ha_reset_result == CONTROL_RESULT_SUCCESS) {
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ } else {
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+ }
+ });
+
+ // The server will only send lease updates if it is not overflown. If
+ // it is overflown, it will rather transition to the waiting state to
+ // initiate full synchronization.
+ if (!overflow) {
+ // Deletions are scheduled first and this should cause the failure.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4"));
+ // This lease update should not be sent because the first update
+ // triggered an error.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3"));
+ }
+
+ if ((partner_state == "load-balancing") || (partner_state == "communication-recovery")) {
+ // The lease updates failed and the partner remains in the load-balancing or
+ // communication-recovery state, so the server should send ha-reset to the
+ // partner to cause it to transition to the waiting state and synchronize
+ // the lease database.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-reset", ""));
+ } else {
+ // The lease updates failed but the partner is already synchronizing lease
+ // database. In this case, don't send the ha-reset.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-reset", ""));
+ }
+
+ // The backlog should be empty.
+ EXPECT_EQ(0, service_->lease_update_backlog_.size());
+ }
+
+ /// @brief Tests scenarios when lease updates are not sent to the failover peer.
+ void testSendUpdatesPartnerDown() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_PARTNER_DOWN_ST));
+
+ // There were no lease updates for which we have been waiting
+ // to complete so the packet was never unparked. Note that in
+ // such situation the packet is not parked either.
+ EXPECT_FALSE(unpark_called);
+
+ // In the partner-down state we don't send lease updates. We
+ // should count transactions for which lease updates were not sent.
+ // This is later returned in the heartbeat so the partner can
+ // determine whether it should synchronize its lease database or
+ // not.
+ EXPECT_EQ(1, service_->communication_state_->getUnsentUpdateCount());
+
+ // Server 2 should not receive lease4-update.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_FALSE(update_request2);
+
+ // Server 2 should not receive lease4-del.
+ auto delete_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_FALSE(delete_request2);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which
+ /// updates are sent is offline.
+ void testSendUpdatesActiveServerOffline() {
+ // Start only two servers out of three. The server 3 is not running.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // Server 2 should not receive lease4-update.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_FALSE(update_request2);
+
+ // Server 2 should not receive lease4-del.
+ auto delete_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_FALSE(delete_request2);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which a
+ /// lease update is sent returns an error.
+ void testSendUpdatesControlResultError() {
+ // Instruct the server 2 to return an error as a result of receiving a command.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // The updates should be sent to server 2 and this server should
+ // return error code.
+ EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Server 2 should receive lease4-update.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request2);
+
+ // Server 2 should receive lease4-del.
+ auto delete_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request2);
+
+ // Lease updates should be successfully sent to server3.
+ EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 3 has received lease4-update command.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request3);
+
+ // Check that the server 3 has received lease4-del command.
+ auto delete_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request3);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which a
+ /// lease update is sent does not authorize the local server.
+ void testSendUpdatesControlResultUnauthorized() {
+ // Instruct the server 2 to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // The updates should be sent to server 2 and this server should
+ // return error code.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+
+ // Lease updates should be successfully sent to server3.
+ EXPECT_EQ(2, factory3_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 3 has received lease4-update command.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request3);
+
+ // Check that the server 3 has received lease4-del command.
+ auto delete_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request3);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which
+ /// updates are sent is offline.
+ void testSendUpdatesBackupServerOffline() {
+ // Start only two servers out of three. The server 2 is not running.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] { unpark_called = true; },
+ false, 1);
+
+ EXPECT_TRUE(unpark_called);
+
+ // The server 2 should have received two commands.
+ EXPECT_EQ(2, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 2 has received lease4-update command.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_TRUE(update_request2);
+
+ // Check that the server 2 has received lease4-del command.
+ auto delete_request2 =
+ factory2_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_TRUE(delete_request2);
+
+ // Server 3 should not receive lease4-update.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-update",
+ "192.1.2.3");
+ EXPECT_FALSE(update_request3);
+
+ // Server 3 should not receive lease4-del.
+ auto delete_request3 =
+ factory3_->getResponseCreator()->findRequest("lease4-del",
+ "192.2.3.4");
+ EXPECT_FALSE(delete_request3);
+ }
+
+ /// @brief Test the scenario when the servers receiving a lease update
+ /// return the conflict status code.
+ void testSendUpdatesControlResultConflict() {
+ // Instruct the server 2 to return an error as a result of receiving a command.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT);
+ factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, false, 1);
+
+ // Ensure that the server has recorded a lease update conflict. The conflict
+ // reported by the backup server should not count.
+ EXPECT_EQ(1, service_->communication_state_->getRejectedLeaseUpdatesCount());
+
+ // Change the partner's response to success.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_SUCCESS);
+
+ // Try sending the lease updates again. The previously rejected lease should
+ // now be accepted and the counter should be 0.
+ bool unpark_called = false;
+ testSendLeaseUpdates([&unpark_called] {
+ unpark_called = true;
+ }, false, 1, MyState(HA_LOAD_BALANCING_ST), true, false);
+ EXPECT_TRUE(unpark_called);
+ EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount());
+ }
+
+ /// @brief Tests scenarios when all lease updates are sent successfully.
+ void testSendSuccessfulUpdates6() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 1);
+
+ // Expecting that the packet was unparked because lease
+ // updates are expected to be successful.
+ EXPECT_TRUE(unpark_called);
+
+ // Updates have been sent so this counter should remain 0.
+ EXPECT_EQ(0, service_->communication_state_->getUnsentUpdateCount());
+
+ // The server 2 should have received one command.
+ EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 2 has received lease6-bulk-apply command.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_TRUE(update_request2);
+
+ // Lease updates should be successfully sent to server3.
+ EXPECT_EQ(1, factory3_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 3 has received lease6-bulk-apply command.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_TRUE(update_request3);
+ }
+
+ /// @brief Tests that DHCPv6 lease updates are queued when the server is in the
+ /// communication-recovery state and later sent before transitioning back to
+ /// the load-balancing state.
+ void testSendUpdatesCommunicationRecovery6() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST));
+
+ // Packet shouldn't be unparked because no updates went out. We merely
+ // queued the updates.
+ EXPECT_FALSE(unpark_called);
+
+ // Let's make sure they have been queued.
+ EXPECT_EQ(2, service_->lease_update_backlog_.size());
+
+ // Make partner available.
+ service_->communication_state_->poke();
+ service_->communication_state_->setPartnerState("load-balancing");
+
+ // This should cause the server to send outstanding lease updates and
+ // because they are all successful the server should transition to the
+ // load-balancing state and continue normal operation.
+ testSynchronousCommands([this]() {
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+ });
+
+ // Bulk lease update should have been sent.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac"));
+
+ // Backlog should be empty.
+ EXPECT_EQ(0, service_->lease_update_backlog_.size());
+ }
+
+ /// @brief Test the cases when the trying to recover from the communication
+ /// interruption and sending lease updates or/and ha-reset fails.
+ ///
+ /// @param partner_state partner state when communication is re-established.
+ /// @param lease_update_result control result returned to lease updates.
+ /// @param ha_reset_result control result returned to ha-reset command.
+ /// @param overflow boolean value indicating if this test should verify the
+ /// case when the leases backlog is overflown (when true), or not (when
+ /// false).
+ void testSendUpdatesCommunicationRecovery6Failed(const std::string& partner_state,
+ const int lease_update_result,
+ const int ha_reset_result,
+ const bool overflow = false) {
+ // Partner responds with a specified control result to lease updates.
+ factory2_->getResponseCreator()->setControlResult("lease6-bulk-apply",
+ lease_update_result);
+ // Partner returns specified control result to ha-reset.
+ factory2_->getResponseCreator()->setControlResult("ha-reset",
+ ha_reset_result);
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_COMMUNICATION_RECOVERY_ST));
+
+ // Packet shouldn't be unparked because no updates went out. We merely
+ // queued the updates.
+ EXPECT_FALSE(unpark_called);
+
+ // Let's make sure they have been queued.
+ EXPECT_EQ(2, service_->lease_update_backlog_.size());
+
+ // When testing the case when the backlog should be overflown, we need
+ // to add several more leases to the backlog to exceed the limit.
+ if (overflow) {
+ ASSERT_NO_THROW(generateTestLeases6());
+ for (auto lease : leases6_) {
+ service_->lease_update_backlog_.push(LeaseUpdateBacklog::ADD, lease);
+ }
+ }
+
+ // Make partner available.
+ service_->communication_state_->poke();
+ service_->communication_state_->setPartnerState(partner_state);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This should cause the server to attempt to send outstanding lease
+ // updates to the partner.
+ testSynchronousCommands([this, ha_reset_result]() {
+ service_->runModel(HAService::NOP_EVT);
+ // If the ha-reset returns success the server should transition to the
+ // waiting state and begin synchronization. Otherwise, if the ha-reset
+ // fails the server should wait in the communication-recovery state
+ // until it succeeds.
+ if (ha_reset_result == CONTROL_RESULT_SUCCESS) {
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ } else {
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+ }
+ });
+
+ // The server will only send lease updates if it is not overflown. If
+ // it is overflown, it will rather transition to the waiting state to
+ // initiate full synchronization.
+ if (!overflow) {
+ // The server should have sent lease updates in a single command.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac"));
+ }
+
+ if ((partner_state == "load-balancing") || (partner_state == "communication-recovery")) {
+ // The lease updates failed and the partner remains in the load-balancing or
+ // communication-recovery state, so the server should send ha-reset to the
+ // partner to cause it to transition to the waiting state and synchronize
+ // the lease database.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-reset", ""));
+ } else {
+ // The lease updates failed but the partner is already synchronizing lease
+ // database. In this case, don't send the ha-reset.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-reset", ""));
+ }
+
+ // Backlog should be empty.
+ EXPECT_EQ(0, service_->lease_update_backlog_.size());
+ }
+
+ /// @brief Tests scenarios when lease updates are not sent to the failover peer.
+ void testSendUpdatesPartnerDown6() {
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 0, MyState(HA_PARTNER_DOWN_ST));
+
+ // There were no lease updates for which we have been waiting to
+ // complete so the packet was never unparked. Note that in such
+ // situation the packet is not parked either.
+ EXPECT_FALSE(unpark_called);
+
+ // In the partner-down state we don't send lease updates. We
+ // should count transactions for which lease updates were not sent.
+ // This is later returned in the heartbeat so the partner can
+ // determine whether it should synchronize its lease database or
+ // not.
+ EXPECT_EQ(1, service_->communication_state_->getUnsentUpdateCount());
+
+ // Server 2 should not receive lease6-bulk-apply.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_FALSE(update_request2);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which
+ /// updates are sent is offline.
+ void testSendUpdatesActiveServerOffline6() {
+ // Start only two servers out of three. The server 3 is not running.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates6([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // Server 2 should not receive lease6-bulk-apply.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_FALSE(update_request2);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which
+ /// updates are sent is offline.
+ void testSendUpdatesBackupServerOffline6() {
+ // Start only two servers out of three. The server 2 is not running.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 1);
+
+ EXPECT_TRUE(unpark_called);
+
+ // The server 2 should have received one command.
+ EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 2 has received lease6-bulk-apply command.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_TRUE(update_request2);
+
+ // Server 3 should not receive lease6-bulk-apply.
+ auto update_request3 =
+ factory3_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_FALSE(update_request3);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which a
+ /// lease update is sent returns an error.
+ void testSendUpdatesControlResultError6() {
+ // Instruct the server 2 to return an error as a result of receiving a command.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates6([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // The updates should be sent to server 2 and this server should return error code.
+ EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Server 2 should receive lease6-bulk-apply.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_TRUE(update_request2);
+ }
+
+ /// @brief Tests scenarios when one of the servers to which a
+ /// lease update is sent does not authorize the local server.
+ void testSendUpdatesControlResultUnauthorized6() {
+ // Instruct the server 2 to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates6([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, true, 1);
+
+ // The updates should be sent to server 2 and this server should return error code.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+ }
+
+ /// @brief These tests verify that the server accepts the response
+ /// to the lease6-bulk-apply command including
+ /// failed-deleted-leases and failed-leases parameters.
+ void testSendUpdatesFailedLeases6() {
+ // Create a dummy lease which failed to be deleted.
+ auto failed_deleted_lease = Element::createMap();
+ failed_deleted_lease->set("type",
+ Element::create("IA_NA"));
+ failed_deleted_lease->set("ip-address",
+ Element::create("2001:db8:1::1"));
+ failed_deleted_lease->set("subnet-id",
+ Element::create(1));
+ failed_deleted_lease->set("result",
+ Element::create(CONTROL_RESULT_EMPTY));
+ failed_deleted_lease->set("error-message",
+ Element::create("no lease found"));
+
+ // Crate a dummy lease which failed to be created.
+ auto failed_lease = Element::createMap();
+ failed_lease->set("type",
+ Element::create("IA_PD"));
+ failed_lease->set("ip-address",
+ Element::create("2001:db8:1::"));
+ failed_lease->set("subnet-id",
+ Element::create(2));
+ failed_lease->set("result",
+ Element::create(CONTROL_RESULT_ERROR));
+ failed_lease->set("error-message",
+ Element::create("failed to create lease"));
+
+ // Create the "failed-deleted-leases" list.
+ auto failed_deleted_leases = Element::createList();
+ failed_deleted_leases->add(failed_deleted_lease);
+
+ // Create the "failed-leases" list.
+ auto failed_leases = Element::createList();
+ failed_leases->add(failed_lease);
+
+ // Add both lists to the arguments.
+ ElementPtr arguments = Element::createMap();
+ arguments->set("failed-deleted-leases", failed_deleted_leases);
+ arguments->set("failed-leases", failed_leases);
+
+ // Configure the server to return this response.
+ factory2_->getResponseCreator()->setArguments("lease6-bulk-apply",
+ arguments);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // This flag will be set to true if unpark is called.
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] { unpark_called = true; },
+ false, 1);
+
+ // Expecting that the packet was unparked because lease
+ // updates are expected to be successful.
+ EXPECT_TRUE(unpark_called);
+
+ // The server 2 should have received one command.
+ EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size());
+
+ // Check that the server 2 has received lease6-bulk-apply command.
+ auto update_request2 =
+ factory2_->getResponseCreator()->findRequest("lease6-bulk-apply",
+ "2001:db8:1::cafe",
+ "2001:db8:1::efac");
+ EXPECT_TRUE(update_request2);
+ }
+
+ /// @brief Test the scenario when the servers receiving a lease update
+ /// return the conflict status code.
+ void testSendUpdatesControlResultConflict6() {
+ // Crate a dummy lease for which a conflict status code is returned.
+ auto failed_lease = Element::createMap();
+ failed_lease->set("type",
+ Element::create("IA_PD"));
+ failed_lease->set("ip-address",
+ Element::create("2001:db8:1::"));
+ failed_lease->set("subnet-id",
+ Element::create(2));
+ failed_lease->set("result",
+ Element::create(CONTROL_RESULT_CONFLICT));
+ failed_lease->set("error-message",
+ Element::create("failed to create lease"));
+
+ // Create the "failed-leases" list.
+ auto failed_leases = Element::createList();
+ failed_leases->add(failed_lease);
+
+ // Add the list to the arguments.
+ ElementPtr arguments = Element::createMap();
+ arguments->set("failed-leases", failed_leases);
+
+ // Active server returns an empty status code to indicate that no lease
+ // has been created. It should cause the HA service to look into the
+ // status codes for individual leases.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_EMPTY);
+ factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_CONFLICT);
+
+ // Configure the server to return our response.
+ factory2_->getResponseCreator()->setArguments("lease6-bulk-apply",
+ arguments);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ testSendLeaseUpdates6([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, false, 1);
+
+ // Ensure that the server has recorded the lease update conflict. The conflict
+ // reported by the backup server should not count.
+ EXPECT_EQ(1, service_->communication_state_->getRejectedLeaseUpdatesCount());
+
+ // Change the active server's response to success. The initially rejected
+ // lease update should no longer be tracked.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_SUCCESS);
+
+ bool unpark_called = false;
+ testSendLeaseUpdates6([&unpark_called] {
+ unpark_called = true;
+ }, false, 1, MyState(HA_LOAD_BALANCING_ST), true, false);
+ EXPECT_TRUE(unpark_called);
+ EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount());
+ }
+
+ /// @brief Test the scenario when the server receiving a lease update returns
+ /// both the conflict and error status code. The latter should take precedence.
+ void testSendUpdatesControlResultConflict6ErrorPrecedence() {
+ // Create the "failed-leases" list.
+ auto failed_leases = Element::createList();
+
+ // Crate a dummy lease for which a conflict status code is returned.
+ auto failed_lease = Element::createMap();
+ failed_lease->set("type",
+ Element::create("IA_NA"));
+ failed_lease->set("ip-address",
+ Element::create("2001:db8:1::"));
+ failed_lease->set("subnet-id",
+ Element::create(2));
+ failed_lease->set("result",
+ Element::create(CONTROL_RESULT_CONFLICT));
+ failed_lease->set("error-message",
+ Element::create("failed to create lease"));
+ failed_leases->add(failed_lease);
+
+ // Create another lease with an error status code.
+ failed_lease = Element::createMap();
+ failed_lease->set("type",
+ Element::create("IA_NA"));
+ failed_lease->set("ip-address",
+ Element::create("2001:db8:1::2"));
+ failed_lease->set("subnet-id",
+ Element::create(2));
+ failed_lease->set("result",
+ Element::create(CONTROL_RESULT_ERROR));
+ failed_lease->set("error-message",
+ Element::create("failed to create lease"));
+ failed_leases->add(failed_lease);
+
+ // Add the list to the arguments.
+ ElementPtr arguments = Element::createMap();
+ arguments->set("failed-leases", failed_leases);
+
+ // Active server returns an empty status code to indicate that no lease
+ // has been created. It should cause the HA service to look into the
+ // status codes for individual leases.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_EMPTY);
+
+ // Configure the server to return our response.
+ factory2_->getResponseCreator()->setArguments("lease6-bulk-apply",
+ arguments);
+
+ // Start HTTP servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ testSendLeaseUpdates6([] {
+ ADD_FAILURE() << "unpark function called but expected that "
+ "the packet is dropped";
+ }, false, 1);
+
+ // The conflict should not be recorded because the error status code
+ // takes precedence.
+ EXPECT_EQ(0, service_->communication_state_->getRejectedLeaseUpdatesCount());
+ }
+
+ /// @brief Runs HAService::processSynchronize for the DHCPv4 server and
+ /// returns a response.
+ ///
+ /// The HAService::processSynchronize is synchronous. Therefore, the IO service
+ /// for HTTP servers is run in a thread. The unit test is responsible for setting
+ /// up the status codes to be returned by the servers, verifying a response and
+ /// leases in the lease database.
+ ///
+ /// @param [out] rsp pointer to the object where response will be stored.
+ void runProcessSynchronize4(ConstElementPtr& rsp) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases4());
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Leases are fetched in pages, so the lease4-get-page should be
+ // sent multiple times. The server is configured to return leases
+ // in 3-element chunks.
+ createPagedSyncResponses4();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in bakckground to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-sync command.
+ ASSERT_NO_THROW(rsp = service.processSynchronize("server2", 20));
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+ }
+
+ /// @brief Runs HAService::processSynchronize for the DHCPv6 server
+ /// and returns a response.
+ ///
+ /// The HAService::processSynchronize is synchronous. Therefore, the IO service
+ /// for HTTP servers is run in a thread. The unit test is responsible for setting
+ /// up the status codes to be returned by the servers, verifying a response and
+ /// leases in the lease database.
+ ///
+ /// @param [out] rsp pointer to the object where response will be stored.
+ void runProcessSynchronize6(ConstElementPtr& rsp) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases6());
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Leases are fetched in pages, so the lease6-get-page should be
+ // sent multiple times. The server is configured to return leases
+ // in 3-element chunks.
+ createPagedSyncResponses6();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in bakckground to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-sync command.
+ ASSERT_NO_THROW(rsp = service.processSynchronize("server2", 20));
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+ }
+
+ /// @brief Pointer to the HA service under test.
+ TestHAServicePtr service_;
+
+ /// @brief HTTP response factory for server 1.
+ TestHttpResponseCreatorFactoryPtr factory_;
+
+ /// @brief HTTP response factory for server 2.
+ TestHttpResponseCreatorFactoryPtr factory2_;
+
+ /// @brief HTTP response factory for server 3.
+ TestHttpResponseCreatorFactoryPtr factory3_;
+
+ /// @brief Test HTTP server 1.
+ HttpListenerPtr listener_;
+
+ /// @brief Test HTTP server 2.
+ HttpListenerPtr listener2_;
+
+ /// @brief Test HTTP server 3.
+ HttpListenerPtr listener3_;
+
+ /// @brief IPv4 leases to be used in the tests.
+ std::vector<Lease4Ptr> leases4_;
+
+ /// @brief IPv6 leases to be used in the tests.
+ std::vector<Lease6Ptr> leases6_;
+
+ /// @brief Basic HTTP authentication user id for server1.
+ std::string user1_;
+
+ /// @brief Basic HTTP authentication password for server1.
+ std::string password1_;
+
+ /// @brief Basic HTTP authentication user id for server2.
+ std::string user2_;
+
+ /// @brief Basic HTTP authentication password for server2.
+ std::string password2_;
+
+ /// @brief Basic HTTP authentication user id for server3.
+ std::string user3_;
+
+ /// @brief Basic HTTP authentication password for server3.
+ std::string password3_;
+};
+
+// Test that server performs load balancing and assigns appropriate classes
+// to the queries.
+TEST_F(HAServiceTest, loadBalancingScopeSelection) {
+ // Create HA configuration for load balancing.
+ HAConfigPtr config_storage = createValidConfiguration();
+ // ... and HA service using this configuration.
+ TestHAService service(io_service_, network_state_, config_storage);
+ service.verboseTransition(HA_LOAD_BALANCING_ST);
+ service.runModel(HAService::NOP_EVT);
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+
+ // Some queries should be processed by this server, some not.
+ if (service.inScope(query4)) {
+ // If the query is to be processed by this server the query
+ // should be assigned to the "HA_server1" class but not to
+ // the "HA_server2" class.
+ ASSERT_TRUE(query4->inClass(ClientClass("HA_server1")));
+ ASSERT_FALSE(query4->inClass(ClientClass("HA_server2")));
+ ++in_scope;
+
+ } else {
+ // If the query is to be processed by another server, the
+ // "HA_server2" class should be assigned instead.
+ ASSERT_FALSE(query4->inClass(ClientClass("HA_server1")));
+ ASSERT_TRUE(query4->inClass(ClientClass("HA_server2")));
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+}
+
+// Test that primary server in hot standby configuration processes all queries.
+TEST_F(HAServiceTest, hotStandbyScopeSelectionThisPrimary) {
+ HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config_storage->getPeerConfig("server2")->setRole("standby");
+
+ // ... and HA service using this configuration.
+ TestHAService service(io_service_, network_state_, config_storage);
+ service.verboseTransition(HA_HOT_STANDBY_ST);
+ service.runModel(HAService::NOP_EVT);
+
+ // Check the reported info about servers.
+ ConstElementPtr ha_servers = service.processStatusGet();
+ ASSERT_TRUE(ha_servers);
+ std::string expected = "{"
+ " \"local\": {"
+ " \"role\": \"primary\","
+ " \"scopes\": [ \"server1\" ],"
+ " \"state\": \"hot-standby\""
+ " }, "
+ " \"remote\": {"
+ " \"age\": 0,"
+ " \"in-touch\": false,"
+ " \"role\": \"standby\","
+ " \"last-scopes\": [ ],"
+ " \"last-state\": \"\","
+ " \"communication-interrupted\": false,"
+ " \"connecting-clients\": 0,"
+ " \"unacked-clients\": 0,"
+ " \"unacked-clients-left\": 0,"
+ " \"analyzed-packets\": 0"
+ " }"
+ "}";
+ EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers));
+
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+
+ // All queries should be processed by this server.
+ ASSERT_TRUE(service.inScope(query4));
+ // The query should be assigned to the "HA_server1" class but not to
+ // the "HA_server2" class.
+ ASSERT_TRUE(query4->inClass(ClientClass("HA_server1")));
+ ASSERT_FALSE(query4->inClass(ClientClass("HA_server2")));
+ }
+}
+
+// Test that secondary server in hot standby configuration processes no queries.
+TEST_F(HAServiceTest, hotStandbyScopeSelectionThisStandby) {
+ HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config_storage->getPeerConfig("server2")->setRole("standby");
+ config_storage->setThisServerName("server2");
+
+ // ... and HA service using this configuration.
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Check the reported info about servers.
+ ConstElementPtr ha_servers = service.processStatusGet();
+ ASSERT_TRUE(ha_servers);
+
+ std::string expected = "{"
+ " \"local\": {"
+ " \"role\": \"standby\","
+ " \"scopes\": [ ],"
+ " \"state\": \"waiting\""
+ " }, "
+ " \"remote\": {"
+ " \"age\": 0,"
+ " \"in-touch\": false,"
+ " \"role\": \"primary\","
+ " \"last-scopes\": [ ],"
+ " \"last-state\": \"\","
+ " \"communication-interrupted\": false,"
+ " \"connecting-clients\": 0,"
+ " \"unacked-clients\": 0,"
+ " \"unacked-clients-left\": 0,"
+ " \"analyzed-packets\": 0"
+ " }"
+ "}";
+ EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers));
+
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+
+ // No queries should be processed by this server.
+ ASSERT_FALSE(service.inScope(query4));
+ // The query should be assigned to the "HA_server1" class but not to
+ // the "HA_server2" class.
+ ASSERT_TRUE(query4->inClass(ClientClass("HA_server1")));
+ ASSERT_FALSE(query4->inClass(ClientClass("HA_server2")));
+ }
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdates) {
+ testSendSuccessfulUpdates();
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdatesMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendSuccessfulUpdates();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state for later send.
+TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery) {
+ testSendUpdatesCommunicationRecovery();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state for later send. Multi threading case.
+TEST_F(HAServiceTest, sendUpdatesCommunicationRecoveryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecovery();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in load-balancing
+// state when the communication is re-established, so the test expects that the
+// ha-reset command is sent to the partner.
+TEST_F(HAServiceTest, communicationRecoveryFailedPartnerLoadBalancing) {
+ testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in load-balancing
+// state when the communication is re-established, so the test expects that the
+// ha-reset command is sent to the partner.
+TEST_F(HAServiceTest, communicationRecoveryFailedPartnerLoadBalancingMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later fails. The partner server in the load-balancing
+// state but sending ha-reset fails. The server should remain in the
+// communication-recovery state.
+TEST_F(HAServiceTest, communicationRecoveryFailedResetFailed) {
+ testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_ERROR);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later fails. The partner server in the load-balancing
+// state but sending ha-reset fails. The server should remain in the
+// communication-recovery state.
+TEST_F(HAServiceTest, communicationRecoveryFailedResetFailedMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecoveryFailed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_ERROR);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in ready state
+// when the communication is re-established, so the test expects that the
+// ha-reset command is NOT sent to the partner. The lease backlog is overflown,
+// so the server should transition to the waiting state.
+TEST_F(HAServiceTest, communicationRecoveryFailedPartnerReady) {
+ testSendUpdatesCommunicationRecoveryFailed("ready", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS, true);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in ready state
+// when the communication is re-established, so the test expects that the
+// ha-reset command is NOT sent to the partner. The lease backlog is overflown,
+// so the server should transition to the waiting state.
+TEST_F(HAServiceTest, communicationRecoveryFailedPartnerReadyMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecoveryFailed("ready", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS, true);
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdatesAuthorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+ testSendSuccessfulUpdates();
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdatesAuthorizedMultiThreading) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+ MultiThreadingMgr::instance().setMode(true);
+ testSendSuccessfulUpdates();
+}
+
+// Test scenario when lease updates are not sent to the failover peer.
+TEST_F(HAServiceTest, sendUpdatesPartnerDown) {
+ testSendUpdatesPartnerDown();
+}
+
+// Test scenario when lease updates are not sent to the failover peer.
+TEST_F(HAServiceTest, sendUpdatesPartnerDownMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesPartnerDown();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesActiveServerOffline) {
+ testSendUpdatesActiveServerOffline();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesActiveServerOfflineMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesActiveServerOffline();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesBackupServerOffline) {
+ testSendUpdatesBackupServerOffline();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesBackupServerOfflineMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesBackupServerOffline();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns an error.
+TEST_F(HAServiceTest, sendUpdatesControlResultError) {
+ testSendUpdatesControlResultError();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns an error.
+TEST_F(HAServiceTest, sendUpdatesControlResultErrorMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultError();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// requires not provided authentication.
+TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized) {
+ testSendUpdatesControlResultUnauthorized();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// requires not provided authentication.
+TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorizedMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultUnauthorized();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns a conflict status code.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflict) {
+ testSendUpdatesControlResultConflict();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns a conflict status code.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflictMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultConflict();
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdates6) {
+ testSendSuccessfulUpdates6();
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdates6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendSuccessfulUpdates6();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state for later send. Then, the partner refuses lease updates causing the
+// server to transition to the waiting state.
+TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery6) {
+ testSendUpdatesCommunicationRecovery6();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state for later send. Then, the partner refuses lease updates causing the
+// server to transition to the waiting state.
+TEST_F(HAServiceTest, sendUpdatesCommunicationRecovery6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecovery6();
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in load-balancing
+// state when the communication is re-established, so the test expects that the
+// ha-reset command is sent to the partner.
+TEST_F(HAServiceTest, communicationRecoveryFailed6PartnerLoadBalancing) {
+ testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in load-balancing
+// state when the communication is re-established, so the test expects that the
+// ha-reset command is sent to the partner.
+TEST_F(HAServiceTest, communicationRecovery6FailedPartnerLoadBalancingMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later fails. The partner server in the load-balancing
+// state but sending ha-reset fails. The server should remain in the
+// communication-recovery state.
+TEST_F(HAServiceTest, communicationRecovery6FailedResetFailed) {
+ testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_ERROR);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later fails. The partner server in the load-balancing
+// state but sending ha-reset fails. The server should remain in the
+// communication-recovery state.
+TEST_F(HAServiceTest, communicationRecovery6FailedResetFailedMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecovery6Failed("load-balancing", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_ERROR);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in ready state
+// when the communication is re-established, so the test expects that the
+// ha-reset command is NOT sent to the partner. The lease backlog is overflown,
+// so the server should transition to the waiting state.
+TEST_F(HAServiceTest, communicationRecovery6FailedPartnerReady) {
+ testSendUpdatesCommunicationRecovery6Failed("ready", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS, true);
+}
+
+// Test scenario when lease updates are queued in the communication-recovery
+// state and sending them later is unsuccessful. Partner is in ready state
+// when the communication is re-established, so the test expects that the
+// ha-reset command is NOT sent to the partner. The lease backlog is overflown,
+// so the server should transition to the waiting state.
+TEST_F(HAServiceTest, communicationRecovery6FailedPartnerReadyMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesCommunicationRecovery6Failed("ready", CONTROL_RESULT_ERROR,
+ CONTROL_RESULT_SUCCESS, true);
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdates6Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+ testSendSuccessfulUpdates6();
+}
+
+// Test scenario when all lease updates are sent successfully.
+TEST_F(HAServiceTest, sendSuccessfulUpdates6AuthorizedMultiThreading) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+ MultiThreadingMgr::instance().setMode(true);
+ testSendSuccessfulUpdates6();
+}
+
+// Test scenario when lease updates are not sent to the failover peer.
+TEST_F(HAServiceTest, sendUpdatesPartnerDown6) {
+ testSendUpdatesPartnerDown6();
+}
+
+// Test scenario when lease updates are not sent to the failover peer.
+TEST_F(HAServiceTest, sendUpdatesPartnerDown6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesPartnerDown6();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesActiveServerOffline6) {
+ testSendUpdatesActiveServerOffline6();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesActiveServerOffline6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesActiveServerOffline6();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesBackupServerOffline6) {
+ testSendUpdatesBackupServerOffline6();
+}
+
+// Test scenario when one of the servers to which updates are sent is offline.
+TEST_F(HAServiceTest, sendUpdatesBackupServerOffline6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesBackupServerOffline6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns an error.
+TEST_F(HAServiceTest, sendUpdatesControlResultError6) {
+ testSendUpdatesControlResultError6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns an error.
+TEST_F(HAServiceTest, sendUpdatesControlResultError6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultError6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// requires not provided authentication.
+TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized6) {
+ testSendUpdatesControlResultUnauthorized6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// requires not provided authentication.
+TEST_F(HAServiceTest, sendUpdatesControlResultUnauthorized6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultUnauthorized6();
+}
+
+// This test verifies that the server accepts the response to the lease6-bulk-apply
+// command including failed-deleted-leases and failed-leases parameters.
+TEST_F(HAServiceTest, sendUpdatesFailedLeases6) {
+ testSendUpdatesFailedLeases6();
+}
+
+// This test verifies that the server accepts the response to the lease6-bulk-apply
+// command including failed-deleted-leases and failed-leases parameters.
+TEST_F(HAServiceTest, sendUpdatesFailedLeases6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesFailedLeases6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns a conflict status code.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflict6) {
+ testSendUpdatesControlResultConflict6();
+}
+
+// Test scenario when one of the servers to which a lease update is sent
+// returns a conflict status code.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflict6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultConflict6();
+}
+
+// Test the scenario when the server receiving a lease update returns
+// both the conflict and error status code. The latter should take precedence.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflict6ErrorPrecedence) {
+ testSendUpdatesControlResultConflict6ErrorPrecedence();
+}
+
+ // Test the scenario when the server receiving a lease update returns
+// both the conflict and error status code. The latter should take precedence.
+TEST_F(HAServiceTest, sendUpdatesControlResultConflict6ErrorPrecedenceMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ testSendUpdatesControlResultConflict6ErrorPrecedence();
+}
+
+// This test verifies that the heartbeat command is processed successfully.
+TEST_F(HAServiceTest, processHeartbeat) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ std::string config_text =
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"load-balancing\","
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:18123/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:18124/\","
+ " \"role\": \"secondary\","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:18125/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " }"
+ " ]"
+ " }"
+ "]";
+
+ // Parse the HA configuration.
+ HAConfigPtr config_storage(new HAConfig());
+ HAConfigParser parser;
+ ASSERT_NO_THROW(parser.parse(config_storage, Element::fromJSON(config_text)));
+
+ TestHAService service(io_service_, network_state_, config_storage);
+ service.query_filter_.serveDefaultScopes();
+
+ int unsent_updates = 6;
+ for (auto i = 0; i < unsent_updates; ++i) {
+ service.communication_state_->increaseUnsentUpdateCount();
+ }
+
+ // Process heartbeat command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processHeartbeat());
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA peer status returned.");
+ // Response must include arguments.
+ ConstElementPtr args = rsp->get("arguments");
+ ASSERT_TRUE(args);
+
+ // Response must include server state as string.
+ ConstElementPtr state = args->get("state");
+ ASSERT_TRUE(state);
+ EXPECT_EQ(Element::string, state->getType());
+ EXPECT_EQ("waiting", state->stringValue());
+
+ // Response must include timestamp when the response was generated.
+ ConstElementPtr date_time = args->get("date-time");
+ ASSERT_TRUE(date_time);
+ EXPECT_EQ(Element::string, date_time->getType());
+
+ auto scopes_list = args->get("scopes");
+ ASSERT_TRUE(scopes_list);
+ EXPECT_EQ(Element::list, scopes_list->getType());
+ ASSERT_EQ(1, scopes_list->size());
+ auto scope = scopes_list->get(0);
+ ASSERT_TRUE(scope);
+ EXPECT_EQ(Element::string, scope->getType());
+ EXPECT_EQ("server1", scope->stringValue());
+
+ // The response should contain the timestamp in the format specified
+ // in RFC1123. We use the HttpDateTime method to parse this timestamp.
+ HttpDateTime t;
+ ASSERT_NO_THROW(t = HttpDateTime::fromRfc1123(date_time->stringValue()));
+
+ // Let's test if the timestamp is accurate. We do it by checking current
+ // time and comparing with the received timestamp.
+ HttpDateTime now;
+ boost::posix_time::time_duration td = now.getPtime() - t.getPtime();
+
+ // Let's allow the response propagation time of 5 seconds to make
+ // sure this test doesn't fail on slow systems.
+ EXPECT_LT(td.seconds(), 5);
+
+ // The response should contain unsent-update-count parameter indicating
+ // how many updates haven't been sent to a partner because the partner
+ // was unavailable.
+ ConstElementPtr unsent_update_count = args->get("unsent-update-count");
+ ASSERT_TRUE(unsent_update_count);
+ EXPECT_EQ(Element::integer, unsent_update_count->getType());
+ EXPECT_EQ(unsent_updates, static_cast<uint64_t>(unsent_update_count->intValue()));
+}
+
+// This test verifies that the correct value of the heartbeat-delay is used.
+TEST_F(HAServiceTest, recurringHeartbeatDelay) {
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Set the heartbeat delay to 6 seconds.
+ config_storage->setHeartbeatDelay(6000);
+
+ // The communication state is the member of the HAServce object. We have to
+ // replace this object with our own implementation to have an ability to
+ // test the setup of the interval timer.
+ NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_,
+ config_storage));
+
+ TestHAService service(io_service_, network_state_, config_storage);
+ service.communication_state_ = state;
+
+ // Let's explicitly transition the state machine to the load balancing state
+ // in which the periodic heartbeats should be generated.
+ ASSERT_NO_THROW(service.verboseTransition(HA_LOAD_BALANCING_ST));
+ ASSERT_NO_THROW(service.runModel(HAService::NOP_EVT));
+
+ ASSERT_TRUE(state->timer_);
+ EXPECT_EQ(6000, state->timer_->getInterval());
+}
+
+// This test verifies that the heartbeat is periodically sent to the
+// other server.
+TEST_F(HAServiceTest, recurringHeartbeat) {
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // All servers are configured to return success and all servers are online.
+ // The heartbeat should be successful (as indicated by the 'true' argument).
+ ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, true));
+
+ // Server 2 should have received the heartbeat
+ EXPECT_GE(factory2_->getResponseCreator()->getReceivedRequests().size(), 0);
+}
+
+// This test verifies that the heartbeat is periodically sent to the
+// other server.
+TEST_F(HAServiceTest, recurringHeartbeatAuthorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // All servers are configured to return success and all servers are online.
+ // The heartbeat should be successful (as indicated by the 'true' argument).
+ ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, true));
+
+ // Server 2 should have received the heartbeat
+ EXPECT_GE(factory2_->getResponseCreator()->getReceivedRequests().size(), 0);
+}
+
+// This test verifies that the heartbeat is considered being unsuccessful if the
+// partner is offline.
+TEST_F(HAServiceTest, recurringHeartbeatServerOffline) {
+ // Start the servers but do not start server 2.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener3_->start();
+ });
+
+ // The servers are configured to return success but the server 2 is offline
+ // so the heartbeat should be unsuccessful.
+ ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_SUCCESS, false));
+
+ // Server 2 is offline so it would be very weird if it received any command.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+}
+
+// This test verifies that the heartbeat is considered being unsuccessful if the
+// partner returns error control result.
+TEST_F(HAServiceTest, recurringHeartbeatControlResultError) {
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // Run the actual test. The servers return a control error and it is expected
+ // that the state is not poked.
+ ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_ERROR, false));
+
+ // Server 2 should have received the heartbeat.
+ EXPECT_EQ(1, factory2_->getResponseCreator()->getReceivedRequests().size());
+}
+
+// This test verifies that the heartbeat is considered being unsuccessful if
+// the partner requires not provided authentication.
+TEST_F(HAServiceTest, recurringHeartbeatControlResultUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ // Run the actual test. The servers return a control error and it is expected
+ // that the state is not poked.
+ ASSERT_NO_FATAL_FAILURE(testRecurringHeartbeat(CONTROL_RESULT_ERROR, false));
+
+ // Server 2 should have received the heartbeat.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+}
+
+// This test verifies that IPv4 leases can be fetched from the peer and inserted
+// or updated in the local lease database.
+TEST_F(HAServiceTest, asyncSyncLeases4) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases4());
+
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ // For every even lease index we add this lease to the database to exercise
+ // the scenario when a lease is already in the database and may be updated
+ // by the lease synchronization procedure.
+ if ((i % 2) == 0) {
+ // Add a copy of the lease to make sure that by modifying the lease
+ // contents we don't affect the lease in the database.
+ Lease4Ptr lease_to_add(new Lease4(*leases4_[i]));
+ // Modify valid lifetime of the lease in the database so we can
+ // later use this value to verify if the lease has been updated.
+ --lease_to_add->valid_lft_;
+ LeaseMgrFactory::instance().addLease(lease_to_add);
+ }
+ }
+
+ // Modify cltt of the first lease. This lease should be updated as a result
+ // of synchronization process because cltt is checked and the lease is
+ // updated if the cltt of the fetched lease is later than the cltt of the
+ // existing lease.
+ ++leases4_[0]->cltt_;
+
+ // For the second lease, set the wrong subnet identifier. This should be
+ // rejected and this lease shouldn't be inserted into the database.
+ // Other leases should be inserted/updated just fine.
+ ++leases4_[1]->subnet_id_ = 0;
+
+ // Modify the partner's lease cltt so it is earlier than the local lease.
+ // Therefore, this lease update should be rejected.
+ --leases4_[2]->cltt_;
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Convert leases to the JSON format, the same as used by the lease_cmds
+ // hook library. Configure our test HTTP servers to return those
+ // leases in this format.
+ ElementPtr response_arguments = Element::createMap();
+
+ // Leases are fetched in pages, so the lease4-get-page should be
+ // sent multiple times. The server is configured to return leases
+ // in 3-element chunks.
+ createPagedSyncResponses4();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() {
+ // Stop running the IO service if we see a lease in the lease
+ // database which is expected to be inserted as a result of lease
+ // syncing.
+ return (!LeaseMgrFactory::instance().getLeases4(SubnetID(10)).empty());
+ }));
+
+ // Check if all leases have been stored in the local database.
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ if (i == 1) {
+ // This lease was purposely malformed and thus shouldn't be
+ // inserted into the database.
+ EXPECT_FALSE(LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_))
+ << "lease " << leases4_[i]->addr_.toText()
+ << " was inserted into the database, but it shouldn't";
+
+ } else {
+ // All other leases should be in the database.
+ Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText()
+ << " not in the lease database";
+ // The lease with #2 returned by the partner is older than its local instance.
+ // The local server should reject this lease.
+ if (i == 2) {
+ // The existing lease should have unmodified timestamp because the
+ // update is expected to be rejected. Same for valid lifetime.
+ EXPECT_LT(leases4_[i]->cltt_, existing_lease->cltt_);
+ EXPECT_NE(leases4_[i]->valid_lft_, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have the same cltt.
+ EXPECT_EQ(leases4_[i]->cltt_, existing_lease->cltt_);
+
+ // Leases with even indexes were added to the database with modified
+ // valid lifetime. Thus the local copy of each such lease should have
+ // this modified valid lifetime. The lease #0 should be updated from
+ // the partner because of the partner's cltt was set to later time.
+ if ((i != 0) && (i % 2) == 0) {
+ EXPECT_EQ(leases4_[i]->valid_lft_ - 1, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have been fetched from the partner and
+ // inserted with no change.
+ EXPECT_EQ(leases4_[i]->valid_lft_, existing_lease->valid_lft_);
+ }
+ }
+ }
+ }
+}
+
+// This test verifies that IPv4 leases can be fetched from the peer and inserted
+// or updated in the local lease database.
+TEST_F(HAServiceTest, asyncSyncLeases4Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases4());
+
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ // For every even lease index we add this lease to the database to exercise
+ // the scenario when a lease is already in the database and may be updated
+ // by the lease synchronization procedure.
+ if ((i % 2) == 0) {
+ // Add a copy of the lease to make sure that by modifying the lease
+ // contents we don't affect the lease in the database.
+ Lease4Ptr lease_to_add(new Lease4(*leases4_[i]));
+ // Modify valid lifetime of the lease in the database so we can
+ // later use this value to verify if the lease has been updated.
+ --lease_to_add->valid_lft_;
+ LeaseMgrFactory::instance().addLease(lease_to_add);
+ }
+ }
+
+ // Modify cltt of the first lease. This lease should be updated as a result
+ // of synchronization process because cltt is checked and the lease is
+ // updated if the cltt of the fetched lease is later than the cltt of the
+ // existing lease.
+ ++leases4_[0]->cltt_;
+
+ // For the second lease, set the wrong subnet identifier. This should be
+ // rejected and this lease shouldn't be inserted into the database.
+ // Other leases should be inserted/updated just fine.
+ ++leases4_[1]->subnet_id_ = 0;
+
+ // Modify the partner's lease cltt so it is earlier than the local lease.
+ // Therefore, this lease update should be rejected.
+ --leases4_[2]->cltt_;
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Convert leases to the JSON format, the same as used by the lease_cmds
+ // hook library. Configure our test HTTP servers to return those
+ // leases in this format.
+ ElementPtr response_arguments = Element::createMap();
+
+ // Leases are fetched in pages, so the lease4-get-page should be
+ // sent multiple times. The server is configured to return leases
+ // in 3-element chunks.
+ createPagedSyncResponses4();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() {
+ // Stop running the IO service if we see a lease in the lease
+ // database which is expected to be inserted as a result of lease
+ // syncing.
+ return (!LeaseMgrFactory::instance().getLeases4(SubnetID(10)).empty());
+ }));
+
+ // Check if all leases have been stored in the local database.
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ if (i == 1) {
+ // This lease was purposely malformed and thus shouldn't be
+ // inserted into the database.
+ EXPECT_FALSE(LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_))
+ << "lease " << leases4_[i]->addr_.toText()
+ << " was inserted into the database, but it shouldn't";
+
+ } else {
+ // All other leases should be in the database.
+ Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText()
+ << " not in the lease database";
+ // The lease with #2 returned by the partner is older than its local instance.
+ // The local server should reject this lease.
+ if (i == 2) {
+ // The existing lease should have unmodified timestamp because the
+ // update is expected to be rejected. Same for valid lifetime.
+ EXPECT_LT(leases4_[i]->cltt_, existing_lease->cltt_);
+ EXPECT_NE(leases4_[i]->valid_lft_, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have the same cltt.
+ EXPECT_EQ(leases4_[i]->cltt_, existing_lease->cltt_);
+
+ // Leases with even indexes were added to the database with modified
+ // valid lifetime. Thus the local copy of each such lease should have
+ // this modified valid lifetime. The lease #0 should be updated from
+ // the partner because of the partner's cltt was set to later time.
+ if ((i != 0) && (i % 2) == 0) {
+ EXPECT_EQ(leases4_[i]->valid_lft_ - 1, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have been fetched from the partner and
+ // inserted with no change.
+ EXPECT_EQ(leases4_[i]->valid_lft_, existing_lease->valid_lft_);
+ }
+ }
+ }
+ }
+}
+
+// Test that there is no exception thrown during leases synchronization
+// when server returns a wrong answer.
+TEST_F(HAServiceTest, asyncSyncLeases4WrongAnswer) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases4());
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Set empty response. This should cause the HA service to log an
+ // error but not crash.
+ ElementPtr response_arguments = Element::createMap();
+
+ factory2_->getResponseCreator()->setArguments(response_arguments);
+ factory3_->getResponseCreator()->setArguments(response_arguments);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// Test that there is no exception thrown during leases synchronization
+// when servers are offline.
+TEST_F(HAServiceTest, asyncSyncLeases4ServerOffline) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service for 1 second.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// Test that there is no exception thrown during leases synchronization
+// when servers require not provided authentication.
+TEST_F(HAServiceTest, asyncSyncLeases4ServerUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=4 type=memfile persist=false"));
+
+ // Create IPv4 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases4());
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service for 1 second.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// This test verifies that IPv6 leases can be fetched from the peer and inserted
+// or updated in the local lease database.
+TEST_F(HAServiceTest, asyncSyncLeases6) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false"));
+
+ // Create IPv6 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases6());
+
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ // For every even lease index we add this lease to the database to exercise
+ // the scenario when a lease is already in the database and may be updated
+ // by the lease synchronization procedure.
+ if ((i % 2) == 0) {
+ // Add a copy of the lease to make sure that by modifying the lease
+ // contents we don't affect the lease in the database.
+ Lease6Ptr lease_to_add(new Lease6(*leases6_[i]));
+ // Modify valid lifetime of the lease in the database so we can
+ // later use this value to verify if the lease has been updated.
+ --lease_to_add->valid_lft_;
+ LeaseMgrFactory::instance().addLease(lease_to_add);
+ }
+ }
+
+ // Modify cltt of the first lease. This lease should be updated as a result
+ // of synchronization process because cltt is checked and the lease is
+ // updated if the cltt of the fetched lease is later than the cltt of the
+ // existing lease.
+ ++leases6_[0]->cltt_;
+
+ // For the second lease, set the wrong subnet identifier. This should be
+ // rejected and this lease shouldn't be inserted into the database.
+ // Other leases should be inserted/updated just fine.
+ ++leases6_[1]->subnet_id_ = 0;
+
+ // Modify the partner's lease cltt so it is earlier than the local lease.
+ // Therefore, this lease update should be rejected.
+ --leases6_[2]->cltt_;
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Convert leases to the JSON format, the same as used by the lease_cmds
+ // hook library. Configure our test HTTP servers to return those
+ // leases in this format.
+ ElementPtr response_arguments = Element::createMap();
+
+ // Leases are fetched in pages, so the lease4-get-page should be
+ // sent multiple times. We need to configure the server side to
+ // return leases in 3-element chunks.
+ createPagedSyncResponses6();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() {
+ // Stop running the IO service if we see a lease in the lease
+ // database which is expected to be inserted as a result of lease
+ // syncing.
+ return (!LeaseMgrFactory::instance().getLeases6(SubnetID(10)).empty());
+ }));
+
+ // Check if all leases have been stored in the local database.
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ if (i == 1) {
+ // This lease was purposely malformed and thus shouldn't be
+ // inserted into the database.
+ EXPECT_FALSE(LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_))
+ << "lease " << leases6_[i]->addr_.toText()
+ << " was inserted into the database, but it shouldn't";
+ } else {
+ // Other leases should be inserted/updated.
+ Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText()
+ << " not in the lease database";
+
+ if (i == 2) {
+ // The existing lease should have unmodified timestamp because the
+ // update is expected to be rejected. Same for valid lifetime.
+ EXPECT_LT(leases6_[i]->cltt_, existing_lease->cltt_);
+ EXPECT_NE(leases6_[i]->valid_lft_, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have the same cltt.
+ EXPECT_EQ(leases6_[i]->cltt_, existing_lease->cltt_);
+
+ // Leases with even indexes were added to the database with modified
+ // valid lifetime. Thus the local copy of each such lease should have
+ // this modified valid lifetime. The lease #0 should be updated from
+ // the partner because of the partner's cltt was set to later time.
+ if ((i != 0) && (i % 2) == 0) {
+ EXPECT_EQ(leases6_[i]->valid_lft_ - 1, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have been fetched from the partner and
+ // inserted with no change.
+ EXPECT_EQ(leases6_[i]->valid_lft_, existing_lease->valid_lft_);
+ }
+ }
+ }
+ }
+}
+
+// This test verifies that IPv6 leases can be fetched from the peer and inserted
+// or updated in the local lease database.
+TEST_F(HAServiceTest, asyncSyncLeases6Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false"));
+
+ // Create IPv6 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases6());
+
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ // For every even lease index we add this lease to the database to exercise
+ // the scenario when a lease is already in the database and may be updated
+ // by the lease synchronization procedure.
+ if ((i % 2) == 0) {
+ // Add a copy of the lease to make sure that by modifying the lease
+ // contents we don't affect the lease in the database.
+ Lease6Ptr lease_to_add(new Lease6(*leases6_[i]));
+ // Modify valid lifetime of the lease in the database so we can
+ // later use this value to verify if the lease has been updated.
+ --lease_to_add->valid_lft_;
+ LeaseMgrFactory::instance().addLease(lease_to_add);
+ }
+ }
+
+ // Modify cltt of the first lease. This lease should be updated as a result
+ // of synchronization process because cltt is checked and the lease is
+ // updated if the cltt of the fetched lease is later than the cltt of the
+ // existing lease.
+ ++leases6_[0]->cltt_;
+
+ // For the second lease, set the wrong subnet identifier. This should be
+ // rejected and this lease shouldn't be inserted into the database.
+ // Other leases should be inserted/updated just fine.
+ ++leases6_[1]->subnet_id_ = 0;
+
+ // Modify the partner's lease cltt so it is earlier than the local lease.
+ // Therefore, this lease update should be rejected.
+ --leases6_[2]->cltt_;
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Convert leases to the JSON format, the same as used by the lease_cmds
+ // hook library. Configure our test HTTP servers to return those
+ // leases in this format.
+ ElementPtr response_arguments = Element::createMap();
+
+ // Leases are fetched in pages, so the lease4-get-page should be
+ // sent multiple times. We need to configure the server side to
+ // return leases in 3-element chunks.
+ createPagedSyncResponses6();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT, []() {
+ // Stop running the IO service if we see a lease in the lease
+ // database which is expected to be inserted as a result of lease
+ // syncing.
+ return (!LeaseMgrFactory::instance().getLeases6(SubnetID(10)).empty());
+ }));
+
+ // Check if all leases have been stored in the local database.
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ if (i == 1) {
+ // This lease was purposely malformed and thus shouldn't be
+ // inserted into the database.
+ EXPECT_FALSE(LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_))
+ << "lease " << leases6_[i]->addr_.toText()
+ << " was inserted into the database, but it shouldn't";
+ } else {
+ // Other leases should be inserted/updated.
+ Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText()
+ << " not in the lease database";
+
+ if (i == 2) {
+ // The existing lease should have unmodified timestamp because the
+ // update is expected to be rejected. Same for valid lifetime.
+ EXPECT_LT(leases6_[i]->cltt_, existing_lease->cltt_);
+ EXPECT_NE(leases6_[i]->valid_lft_, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have the same cltt.
+ EXPECT_EQ(leases6_[i]->cltt_, existing_lease->cltt_);
+
+ // Leases with even indexes were added to the database with modified
+ // valid lifetime. Thus the local copy of each such lease should have
+ // this modified valid lifetime. The lease #0 should be updated from
+ // the partner because of the partner's cltt was set to later time.
+ if ((i != 0) && (i % 2) == 0) {
+ EXPECT_EQ(leases6_[i]->valid_lft_ - 1, existing_lease->valid_lft_);
+
+ } else {
+ // All other leases should have been fetched from the partner and
+ // inserted with no change.
+ EXPECT_EQ(leases6_[i]->valid_lft_, existing_lease->valid_lft_);
+ }
+ }
+ }
+ }
+}
+
+// Test that there is no exception thrown during IPv6 leases synchronization
+// when server returns a wrong answer.
+TEST_F(HAServiceTest, asyncSyncLeases6WrongAnswer) {
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false"));
+
+ // Create IPv6 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases6());
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Set empty response. This should cause the HA service to log an
+ // error but not crash.
+ ElementPtr response_arguments = Element::createMap();
+
+ factory2_->getResponseCreator()->setArguments(response_arguments);
+ factory3_->getResponseCreator()->setArguments(response_arguments);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// Test that there is no exception thrown during IPv6 leases synchronization
+// when servers are offline.
+TEST_F(HAServiceTest, asyncSyncLeases6ServerOffline) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ TestHAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service for 1 second.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// Test that there is no exception thrown during IPv6 leases synchronization
+// when server requires not provided authentication.
+TEST_F(HAServiceTest, asyncSyncLeases6Unauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create lease manager.
+ ASSERT_NO_THROW(LeaseMgrFactory::create("universe=6 type=memfile persist=false"));
+
+ // Create IPv6 leases which will be fetched from the other server.
+ ASSERT_NO_THROW(generateTestLeases6());
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Setting the heartbeat delay to 0 disables the recurring heartbeat.
+ // We just want to synchronize leases and not send the heartbeat.
+ config_storage->setHeartbeatDelay(0);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage,
+ HAServerType::DHCPv6);
+
+ // Start fetching leases asynchronously.
+ ASSERT_NO_THROW(service.asyncSyncLeases());
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(1000));
+}
+
+// This test verifies that the ha-sync command is processed successfully for the
+// DHCPv4 server.
+TEST_F(HAServiceTest, processSynchronize4) {
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate success.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization"
+ " complete.");
+
+ // All leases should have been inserted into the database.
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText()
+ << " not in the lease database";
+ }
+
+ // The following commands should have been sent to the server2: dhcp-disable,
+ // lease4-get-page and ha-sync-complete-notify.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", ""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable", ""));
+}
+
+// This test verifies that the ha-sync command is processed successfully for the
+// DHCPv4 server.
+TEST_F(HAServiceTest, processSynchronize4Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate success.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization"
+ " complete.");
+
+ // All leases should have been inserted into the database.
+ for (size_t i = 0; i < leases4_.size(); ++i) {
+ Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(leases4_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases4_[i]->addr_.toText()
+ << " not in the lease database";
+ }
+
+ // The following commands should have been sent to the server2: dhcp-disable,
+ // lease4-get-page and ha-sync-complete-notify.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", ""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable", ""));
+}
+
+// This test verifies that an error is reported when sending a dhcp-disable
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronizeDisableError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("dhcp-disable",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should only receive dhcp-disable command. Remaining three should
+ // not be sent.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease4-get-page",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that an error is reported when sending any not
+// authenticated command causes a not authorized error.
+TEST_F(HAServiceTest, processSynchronizeUnauthorized) {
+ // Instruct server2 to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should only receive dhcp-disable commands.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+}
+
+// This test verifies that an error is reported when sending a lease4-get-page
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronizeLease4GetPageError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("lease4-get-page",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should receive dhcp-disable, lease4-get-page and dhcp-enable
+ // commands. The lease4-get-page command results in an error (configured at
+ // the beginning of this test), but the dhcp-enable command should be sent
+ // to enable the DHCP service after the error anyway. The
+ // ha-sync-complete-notify must not be sent after the error. It should only
+ // be sent after successful synchronization.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease4-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+}
+
+// This test verifies that an error is reported when sending a dhcp-enable
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronizeEnableError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("dhcp-enable",
+ CONTROL_RESULT_ERROR);
+
+ // Return the unsupported command status for this command to enforce
+ // sending the dhcp-enable command.
+ factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify",
+ CONTROL_RESULT_COMMAND_UNSUPPORTED);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should receive four commands of which ha-sync-complete-notify
+ // was unsupported.
+
+ auto requests = factory2_->getResponseCreator()->getReceivedRequests();
+ ASSERT_GE(requests.size(), 3);
+
+ // The dhcp-disable should be the first request.
+ auto request = factory2_->getResponseCreator()->findRequest("dhcp-disable","20");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[0], request);
+
+ // There may be multiple lease4-get-page commands but the first one should
+ // follow the dhcp-disable request.
+ request = factory2_->getResponseCreator()->findRequest("lease4-get-page", "");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[1], request);
+
+ // The ha-sync-complete-notify should be last but one.
+ request = factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify", "");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[requests.size()-2], request);
+
+ // The dhcp-enable should be last.
+ request = factory2_->getResponseCreator()->findRequest("dhcp-enable", "");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[requests.size()-1], request);
+}
+
+// This test verifies that dhcp-enable command is not sent to the partner after
+// receiving an error to the ha-sync-complete-notify command.
+TEST_F(HAServiceTest, processSynchronizeNotifyError) {
+ // Return an error to the ha-sync-complete-notify command.
+ factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize4(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ auto requests = factory2_->getResponseCreator()->getReceivedRequests();
+
+ ASSERT_GE(requests.size(), 3);
+
+ // The dhcp-disable should be the first request.
+ auto request = factory2_->getResponseCreator()->findRequest("dhcp-disable","20");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[0], request);
+
+ // There may be multiple lease4-get-page commands but the first one should
+ // follow the dhcp-disable request.
+ request = factory2_->getResponseCreator()->findRequest("lease4-get-page","");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(requests[1], request);
+
+ // The ha-sync-complete-notify should be last.
+ request = factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify","");
+ ASSERT_TRUE(request);
+ EXPECT_EQ(*(requests.rbegin()), request);
+
+ // The dhcp-enable should not be sent after ha-sync-complete-notify failure.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that the ha-sync command is processed successfully for the
+// DHCPv6 server.
+TEST_F(HAServiceTest, processSynchronize6) {
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate success.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization"
+ " complete.");
+
+ // All leases should have been inserted into the database.
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText()
+ << " not in the lease database";
+ }
+
+ // The following commands should have been sent to the server2: dhcp-disable,
+ // lease6-get-page and ha-sync-complete-notify.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+}
+
+// This test verifies that the ha-sync command is processed successfully for the
+// DHCPv6 server.
+TEST_F(HAServiceTest, processSynchronize6Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate success.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Lease database synchronization"
+ " complete.");
+
+ // All leases should have been inserted into the database.
+ for (size_t i = 0; i < leases6_.size(); ++i) {
+ Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(Lease::TYPE_NA,
+ leases6_[i]->addr_);
+ ASSERT_TRUE(existing_lease) << "lease " << leases6_[i]->addr_.toText()
+ << " not in the lease database";
+ }
+
+ // The following commands should have been sent to the server2: dhcp-disable,
+ // lease6-get-page and ha-sync-complete-notify.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+}
+
+// This test verifies that an error is reported when sending a dhcp-disable
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronize6DisableError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("dhcp-disable",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should only receive dhcp-disable command.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that an error is reported when sending any not
+// authenticated command causes a not authorized error.
+TEST_F(HAServiceTest, processSynchronize6Unauthorized) {
+ // Instruct server2 to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should only receive dhcp-disable commands.
+ EXPECT_TRUE(factory2_->getResponseCreator()->getReceivedRequests().empty());
+}
+
+// This test verifies that an error is reported when sending a lease6-get-page
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronizeLease6GetPageError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("lease6-get-page",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should receive all commands. The dhcp-disable was successful, so
+ // the dhcp-enable command must be sent to re-enable the service after failure.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+}
+
+// This test verifies that an error is reported when sending a dhcp-enable
+// command causes an error.
+TEST_F(HAServiceTest, processSynchronize6EnableError) {
+ // Setup the server2 to return an error to dhcp-disable commands.
+ factory2_->getResponseCreator()->setControlResult("dhcp-enable",
+ CONTROL_RESULT_ERROR);
+
+ // Return the unsupported command status for this command to enforce
+ // sending the dhcp-enable command.
+ factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify",
+ CONTROL_RESULT_COMMAND_UNSUPPORTED);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ // The server2 should receive four commands of which ha-sync-complete-notify
+ // was unsupported.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that dhcp-enable command is not sent to the partner after
+// receiving an error to the ha-sync-complete-notify command.
+TEST_F(HAServiceTest, processSynchronize6NotifyError) {
+ // Return an error to the ha-sync-complete-notify command.
+ factory2_->getResponseCreator()->setControlResult("ha-sync-complete-notify",
+ CONTROL_RESULT_ERROR);
+
+ // Run HAService::processSynchronize and gather a response.
+ ConstElementPtr rsp;
+ runProcessSynchronize6(rsp);
+
+ // The response should indicate an error
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR);
+
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-disable","20"));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("lease6-get-page",""));
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("ha-sync-complete-notify",""));
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that the DHCPv4 service can be disabled on the remote server.
+TEST_F(HAServiceTest, asyncDisableDHCPService4) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-disable command with max-period of 10 seconds.
+ // When the transaction is finished, the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10,
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_TRUE(success);
+ EXPECT_TRUE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+
+ // The second server should not receive the command.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-disable","10"));
+ // The third server should receive the dhcp-disable command with the max-period
+ // value of 10.
+ EXPECT_TRUE(factory3_->getResponseCreator()->findRequest("dhcp-disable","10"));
+}
+
+// This test verifies that the DHCPv4 service can be disabled on the remote server.
+TEST_F(HAServiceTest, asyncDisableDHCPService4Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-disable command with max-period of 10 seconds.
+ // When the transaction is finished, the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10,
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_TRUE(success);
+ EXPECT_TRUE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+
+ // The second server should not receive the command.
+ EXPECT_FALSE(factory2_->getResponseCreator()->findRequest("dhcp-disable","10"));
+ // The third server should receive the dhcp-disable command with the max-period
+ // value of 10.
+ EXPECT_TRUE(factory3_->getResponseCreator()->findRequest("dhcp-disable","10"));
+}
+
+// This test verifies that there is no exception thrown as a result of dhcp-disable
+// command when the server is offline.
+TEST_F(HAServiceTest, asyncDisableDHCPService4ServerOffline) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-disable command with max-period of 10 seconds.
+ // When the transaction is finished, the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncDisableDHCPService("server2", 10,
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that an error is returned when the remote server
+// returns control status error.
+TEST_F(HAServiceTest, asyncDisableDHCPService4ControlResultError) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Set the servers to return error code in response to the dhcp-enable
+ // command.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+ factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-disable command with max-period of 10 seconds.
+ // When the transaction is finished, the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10,
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that an error is returned when the remote server
+// requires not provided authentication.
+TEST_F(HAServiceTest, asyncDisableDHCPService4ControlResultUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-disable command with max-period of 10 seconds.
+ // When the transaction is finished, the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncDisableDHCPService("server3", 10,
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that the DHCPv4 service can be enabled on the remote server.
+TEST_F(HAServiceTest, asyncEnableDHCPService4) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-enable command. When the transaction is finished,
+ // the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncEnableDHCPService("server2",
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_TRUE(success);
+ EXPECT_TRUE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+
+ // The second server should receive the dhcp-enable.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+ // The third server should not receive the command.
+ EXPECT_FALSE(factory3_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that the DHCPv4 service can be enabled on the remote server.
+TEST_F(HAServiceTest, asyncEnableDHCPService4Authorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-enable command. When the transaction is finished,
+ // the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncEnableDHCPService("server2",
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_TRUE(success);
+ EXPECT_TRUE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+
+ // The second server should receive the dhcp-enable.
+ EXPECT_TRUE(factory2_->getResponseCreator()->findRequest("dhcp-enable",""));
+ // The third server should not receive the command.
+ EXPECT_FALSE(factory3_->getResponseCreator()->findRequest("dhcp-enable",""));
+}
+
+// This test verifies that there is no exception thrown as a result of dhcp-enable
+// command when the server is offline.
+TEST_F(HAServiceTest, asyncEnableDHCPService4ServerOffline) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-enable command. When the transaction is finished,
+ // the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncEnableDHCPService("server2",
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that an error is returned when the remote server
+// returns control status error.
+TEST_F(HAServiceTest, asyncEnableDHCPService4ControlResultError) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Set the servers to return error code in response to the dhcp-enable
+ // command.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+ factory3_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-enable command. When the transaction is finished,
+ // the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncEnableDHCPService("server2",
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that an error is returned when the remote server
+// requires not provided authentication.
+TEST_F(HAServiceTest, asyncEnableDHCPService4ControlResultUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ listener3_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Send dhcp-enable command. When the transaction is finished,
+ // the IO service gets stopped.
+ ASSERT_NO_THROW(service.asyncEnableDHCPService("server2",
+ [this](const bool success,
+ const std::string& error_message,
+ const int) {
+ EXPECT_FALSE(success);
+ EXPECT_FALSE(error_message.empty());
+ io_service_->stop();
+ }));
+
+ // Run IO service to actually perform the transaction.
+ ASSERT_NO_THROW(runIOService(TEST_TIMEOUT));
+}
+
+// This test verifies that the "ha-scopes" command is processed correctly.
+TEST_F(HAServiceTest, processScopes) {
+ // Create HA configuration.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Create HA service using this configuration.
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Enable "server1" and "server2" scopes.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processScopes({ "server1", "server2" }));
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured.");
+
+ // Verify that "server1" and "server2" scopes are enabled.
+ EXPECT_TRUE(service.query_filter_.amServingScope("server1"));
+ EXPECT_TRUE(service.query_filter_.amServingScope("server2"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server3"));
+
+ // Enable "server2" scope only.
+ ASSERT_NO_THROW(rsp = service.processScopes({ "server2" }));
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured.");
+ EXPECT_FALSE(service.query_filter_.amServingScope("server1"));
+ EXPECT_TRUE(service.query_filter_.amServingScope("server2"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server3"));
+
+ // Clear scopes.
+ ASSERT_NO_THROW(rsp = service.processScopes({ }));
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "New HA scopes configured.");
+ EXPECT_FALSE(service.query_filter_.amServingScope("server1"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server2"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server3"));
+
+ // Unsupported scope.
+ ASSERT_NO_THROW(rsp = service.processScopes({ "server1", "unsupported", "server3" }));
+ checkAnswer(rsp, CONTROL_RESULT_ERROR, "invalid server name specified 'unsupported'"
+ " while enabling/disabling HA scopes");
+ // Even though the "server1" is a valid scope name, it should not be
+ // enabled because we expect scopes enabling to be atomic operation,
+ // i.e. all or nothing.
+ EXPECT_FALSE(service.query_filter_.amServingScope("server1"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server2"));
+ EXPECT_FALSE(service.query_filter_.amServingScope("server3"));
+}
+
+// This test verifies that the ha-continue command is processed successfully.
+TEST_F(HAServiceTest, processContinue) {
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // State machine is to be paused in the waiting state.
+ ASSERT_NO_THROW(config_storage->getStateMachineConfig()->
+ getStateConfig(HA_WAITING_ST)->setPausing("always"));
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Pause the state machine.
+ EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT));
+ EXPECT_TRUE(service.isModelPaused());
+
+ // Process ha-continue command that should unpause the state machine.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processContinue());
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine continues.");
+
+ // State machine should have been unpaused as a result of processing the
+ // command.
+ EXPECT_FALSE(service.isModelPaused());
+
+ // Response should include no arguments.
+ EXPECT_FALSE(rsp->get("arguments"));
+
+ // Sending ha-continue command again which should return success but
+ // slightly different textual status.
+ ASSERT_NO_THROW(rsp = service.processContinue());
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine is not paused.");
+
+ // The state machine should not be paused.
+ EXPECT_FALSE(service.isModelPaused());
+
+ // Response should include no arguments.
+ EXPECT_FALSE(rsp->get("arguments"));
+}
+
+// This test verifies that the ha-maintenance-notify command is processed
+// successfully and transitions the state machine to the in-maintenance state
+// and that it is possible to revert to the previous state.
+// It also verifies that this command fails to transition the state machine
+// for some selected states for which it is unsupported.
+TEST_F(HAServiceTest, processMaintenanceNotify) {
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ std::set<int> valid_states = {
+ HA_WAITING_ST,
+ HA_READY_ST,
+ HA_SYNCING_ST,
+ HA_PARTNER_DOWN_ST,
+ HA_LOAD_BALANCING_ST,
+ HA_HOT_STANDBY_ST,
+ };
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Test transition from the states for which it is allowed.
+ for (auto state : valid_states) {
+ EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT));
+
+ // Process ha-maintenance-notify command that should transition the
+ // state machine to the in-maintenance state.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false));
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state.");
+
+ // The state machine should have been transitioned to the in-maintenance state.
+ EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState());
+
+ // Try to cancel the maintenance.
+ ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true));
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance canceled.");
+
+ // The state machine should have been transitioned to the state it was in
+ // prior to transitioning to the in-maintenance state.
+ EXPECT_EQ(state, service.getCurrState());
+ }
+
+ std::set<int> invalid_states = {
+ HA_TERMINATED_ST,
+ HA_BACKUP_ST,
+ HA_PARTNER_IN_MAINTENANCE_ST
+ };
+
+ // Make sure that the transition from the other states is not allowed.
+ for (auto state : invalid_states) {
+ EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT));
+ EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT));
+
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false));
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED,
+ "Unable to transition the server from the "
+ + stateToString(state) + " to in-maintenance state.");
+ }
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-start
+// command successfully transitions to the partner-in-maintenance state and its
+// partner transitions to the in-maintenance state.
+TEST_F(HAServiceTest, processMaintenanceStartSuccess) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-in-maintenance state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is now in the partner-in-maintenance state"
+ " and its partner is in-maintenance state. The partner can be now safely"
+ " shut down.");
+
+ EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-start
+// command successfully transitions to the partner-in-maintenance state and its
+// partner transitions to the in-maintenance state.
+TEST_F(HAServiceTest, processMaintenanceStartSuccessAuthorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-in-maintenance state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is now in the partner-in-maintenance state"
+ " and its partner is in-maintenance state. The partner can be now safely"
+ " shut down.");
+
+ EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState());
+}
+
+// This test verifies the case that the server transitions to the partner-down
+// state after receiving the ha-maintenance-start command. This is the case
+// when the communication with the partner server fails while this command
+// is received. It is assumed that the partner server is already terminated
+// for maintenance.
+TEST_F(HAServiceTest, processMaintenanceStartPartnerDown) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the server, but don't start the partner. This simulates
+ // the case that the partner is already down for maintenance.
+ ASSERT_NO_THROW({
+ listener_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-in-maintenance state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS,
+ "Server is now in the partner-down state as its"
+ " partner appears to be offline for maintenance.");
+
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server is receiving
+// ha-maintenance-start command and tries to notify the partner
+// which returns an error.
+TEST_F(HAServiceTest, processMaintenanceStartPartnerError) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Simulate an error returned by the partner.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS,
+ "Server is now in the partner-down state as its"
+ " partner appears to be offline for maintenance.");
+
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server is receiving
+// ha-maintenance-start command and tries to notify the partner
+// which requires not provided authentication.
+TEST_F(HAServiceTest, processMaintenanceStartPartnerUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS,
+ "Server is now in the partner-down state as its"
+ " partner appears to be offline for maintenance.");
+
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server is receiving
+// ha-maintenance-start command and tries to notify the partner
+// which returns a special result indicating that it can't enter
+// the in-maintenance state.
+TEST_F(HAServiceTest, processMaintenanceStartNotAllowed) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Simulate an error returned by the partner.
+ factory2_->getResponseCreator()->
+ setControlResult(TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ HAService service(io_service_, network_state_, config_storage);
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-start command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceStart());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR,
+ "Unable to transition to the partner-in-maintenance state."
+ " The partner server responded with the following message"
+ " to the ha-maintenance-notify command: response returned"
+ " (error code 1001).");
+
+ // The partner's state precludes entering the in-maintenance state. Thus, this
+ // server must not change its state either.
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-cancel
+// command successfully transitions out of the partner-in-maintenance state.
+TEST_F(HAServiceTest, processMaintenanceCancelSuccess) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-in-maintenance state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
+
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-cancel
+// command successfully transitions out of the partner-in-maintenance state.
+TEST_F(HAServiceTest, processMaintenanceCancelSuccessAuthorized) {
+ // Update config to provide authentication.
+ user2_ = "foo";
+ password2_ = "bar";
+ user3_ = "test";
+ password3_ = "1234";
+
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+ setBasicAuth(config_storage);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner of our server is online and should have responded with
+ // the success status. Therefore, this server should have transitioned
+ // to the partner-in-maintenance state.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
+
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies that the maintenance is not canceled in case the
+// partner returns an error.
+TEST_F(HAServiceTest, processMaintenanceCancelPartnerError) {
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Simulate an error returned by the partner.
+ factory2_->getResponseCreator()->setControlResult(CONTROL_RESULT_ERROR);
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner should have responded with an error.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR,
+ "Unable to cancel maintenance. The partner server responded"
+ " with the following message to the ha-maintenance-notify"
+ " command: response returned (error code 1).");
+
+ // The state of this server should not change.
+ EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState());
+}
+
+// This test verifies that the maintenance is not canceled in case the
+// partner requires not provided authentication.
+TEST_F(HAServiceTest, processMaintenanceCancelPartnerUnauthorized) {
+ // Instruct servers to require authentication.
+ factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
+ factory3_->getResponseCreator()->addBasicAuth("test", "1234");
+
+ // Create HA configuration for 3 servers. This server is
+ // server 1.
+ HAConfigPtr config_storage = createValidConfiguration();
+
+ // Start the servers.
+ ASSERT_NO_THROW({
+ listener_->start();
+ listener2_->start();
+ });
+
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ // The tested function is synchronous, so we need to run server side IO service
+ // in background to not block the main thread.
+ auto thread = runIOServiceInThread();
+
+ // Process ha-maintenance-cancel command.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+ io_service_->get_io_service().reset();
+ io_service_->poll();
+
+ // The partner should have responded with an error.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_ERROR,
+ "Unable to cancel maintenance. The partner server responded"
+ " with the following message to the ha-maintenance-notify"
+ " command: Unauthorized (error code 1).");
+
+ // The state of this server should not change.
+ EXPECT_EQ(HA_PARTNER_IN_MAINTENANCE_ST, service.getCurrState());
+}
+
+// This test verifies that the ha-reset command is processed successfully.
+TEST_F(HAServiceTest, processHAReset) {
+ HAConfigPtr config_storage = createValidConfiguration();
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Transition the server to the load-balancing state.
+ EXPECT_NO_THROW(service.transition(HA_LOAD_BALANCING_ST, HAService::NOP_EVT));
+
+ // Process ha-reset command that should cause the server to transition
+ // to the waiting state.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processHAReset());
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine reset.");
+
+ // Response should include no arguments.
+ EXPECT_FALSE(rsp->get("arguments"));
+
+ // The server should be in the waiting state.
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies that the ha-reset command is processed successfully when
+// the server is already in the waiting state.
+TEST_F(HAServiceTest, processHAResetWaiting) {
+ HAConfigPtr config_storage = createValidConfiguration();
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Transition the server to the waiting state.
+ EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT));
+
+ // Process ha-reset command that should not change the state of the
+ // server because the server is already in the waiting state. It
+ // should not fail though.
+ ConstElementPtr rsp;
+ ASSERT_NO_THROW(rsp = service.processHAReset());
+
+ // The server should have responded.
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state.");
+
+ // Response should include no arguments.
+ EXPECT_FALSE(rsp->get("arguments"));
+
+ // The server should be in the waiting state.
+ EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies that the ha-sync-complete-notify command is processed
+// successfully, the server keeps the DHCP service disabled in the partner-down
+// state and enables the service when it is in another state.
+TEST_F(HAServiceTest, processSyncCompleteNotify) {
+ HAConfigPtr config_storage = createValidConfiguration();
+ TestHAService service(io_service_, network_state_, config_storage);
+
+ // Transition the server to the partner-down state.
+ EXPECT_NO_THROW(service.transition(HA_PARTNER_DOWN_ST, HAService::NOP_EVT));
+
+ // Simulate disabling the DHCP service for synchronization.
+ EXPECT_NO_THROW(service.network_state_->disableService(NetworkState::Origin::HA_COMMAND));
+
+ ConstElementPtr rsp;
+ EXPECT_NO_THROW(rsp = service.processSyncCompleteNotify());
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS,
+ "Server successfully notified about the synchronization completion.");
+
+ // The server should remain in the partner-down state.
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service.getCurrState());
+
+ // The service should be disabled until the server transitions to the
+ // normal state.
+ EXPECT_FALSE(service.network_state_->isServiceEnabled());
+
+ // It is possible that the connection from this server to the partner
+ // is still broken. In that case, the HA_SYNCED_PARTNER_UNAVAILABLE_EVT
+ // is emitted and the server should enable DHCP service to continue
+ // serving the clients in the partner-down state.
+ EXPECT_NO_THROW(service.postNextEvent(HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT));
+ EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT));
+ EXPECT_TRUE(service.network_state_->isServiceEnabled());
+
+ // Transition the server to the load-balancing state.
+ EXPECT_NO_THROW(service.transition(HA_LOAD_BALANCING_ST, HAService::NOP_EVT));
+
+ // Disable the service again.
+ EXPECT_NO_THROW(service.network_state_->disableService(NetworkState::Origin::HA_COMMAND));
+
+ EXPECT_NO_THROW(rsp = service.processSyncCompleteNotify());
+
+ ASSERT_TRUE(rsp);
+ checkAnswer(rsp, CONTROL_RESULT_SUCCESS,
+ "Server successfully notified about the synchronization completion.");
+
+ // The server should remain in the load-balancing state.
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service.getCurrState());
+
+ // This time the service should be enabled because the server is in
+ // the state in which it sends the lease updates.
+ EXPECT_TRUE(service.network_state_->isServiceEnabled());
+}
+
+/// @brief HA partner to the server under test.
+///
+/// This is a wrapper class around @c HttpListener which simulates a
+/// partner server. It provides convenient methods to start, stop the
+/// partner (its listener) and to transition the partner between various
+/// HA states. Depending on the state and whether the partner is started
+/// or stopped, different answers are returned in response to the
+/// ha-heartbeat commands.
+class HAPartner {
+public:
+
+ /// @brief Constructor.
+ ///
+ /// Creates the partner instance from a listner and the corresponding
+ /// response factory. It automatically transitions the partner to the
+ /// "waiting" state unless otherwise specified with the third parameter.
+ ///
+ /// @param listner pointer to the listner to be used.
+ /// @param factory pointer to the response factory to be used. This
+ /// must be the same factory that the listener is using.
+ /// @param initial_state initial state for the partner. Default is to
+ /// transition the partner to the "waiting" state which is the default
+ /// state for each starting server.
+ HAPartner(const HttpListenerPtr& listener,
+ const TestHttpResponseCreatorFactoryPtr& factory,
+ const std::string& initial_state = "waiting")
+ : listener_(listener), factory_(factory), running_(false),
+ static_date_time_(), static_scopes_(),
+ static_unsent_update_count_(0) {
+ transition(initial_state);
+ }
+
+ /// @brief Sets control result to be returned as a result of the
+ /// communication with the partner.
+ ///
+ /// @param control_result new control result value.
+ void setControlResult(const int control_result) {
+ factory_->getResponseCreator()->setControlResult(control_result);
+ }
+
+ /// @brief Sets static date-time value to be used in responses.
+ ///
+ /// @param static_date_time fixed date-time value.
+ void setDateTime(const std::string& static_date_time) {
+ static_date_time_ = static_date_time;
+ }
+
+ /// @brief Sets static scopes to be used in responses.
+ ///
+ /// @param scopes Fixed scopes set.
+ void setScopes(const std::set<std::string>& scopes) {
+ static_scopes_ = scopes;
+ }
+
+ /// @brief Enable response to commands required for leases synchronization.
+ ///
+ /// Enables dhcp-disable, dhcp-enable and lease4-get-page commands. The last
+ /// of them returns a bunch of test leases.
+ void enableRespondLeaseFetching() {
+ // Create IPv4 leases which will be fetched from the other server.
+ std::vector<Lease4Ptr> leases4;
+ ASSERT_NO_THROW(generateTestLeases(leases4));
+
+ // Convert leases to the JSON format, the same as used by the lease_cmds
+ // hook library. Configure our test HTTP servers to return those
+ // leases in this format.
+ ElementPtr response_arguments = Element::createMap();
+ response_arguments->set("leases", getLeasesAsJson(leases4));
+
+ factory_->getResponseCreator()->setArguments("lease4-get-page", response_arguments);
+ }
+
+ /// @brief Starts up the partner.
+ void startup() {
+ if (!running_) {
+ listener_->start();
+ running_ = true;
+ }
+ }
+
+ /// @brief Shuts down the partner.
+ ///
+ /// It may be used to simulate partner's crash as well as graceful
+ /// shutdown.
+ void shutdown() {
+ if (running_) {
+ listener_->stop();
+ running_ = false;
+ }
+ }
+
+ /// @brief Transitions the partner to the specified state.
+ ///
+ /// The state is provided in the textual form and the function doesn't
+ /// validate whether it is correct or not.
+ ///
+ /// @param state new partner state.
+ void transition(const std::string& state) {
+ ElementPtr response_arguments = Element::createMap();
+ response_arguments->set("state", Element::create(state));
+ if (!static_date_time_.empty()) {
+ response_arguments->set("date-time", Element::create(static_date_time_));
+ }
+ if (!static_scopes_.empty()) {
+ auto json_scopes = Element::createList();
+ for (auto scope : static_scopes_) {
+ json_scopes->add(Element::create(scope));
+ }
+ response_arguments->set("scopes", json_scopes);
+ }
+ if (static_unsent_update_count_ >= 0) {
+ response_arguments->set("unsent-update-count", Element::create(static_unsent_update_count_));
+ }
+ factory_->getResponseCreator()->setArguments(response_arguments);
+ }
+
+ /// @brief Sets static value of unsent update count.
+ ///
+ /// @param unsent_update_count new value of the unsent update count.
+ /// Specify a negative value to exclude the count from the response.
+ void setUnsentUpdateCount(int64_t unsent_update_count) {
+ static_unsent_update_count_ = unsent_update_count;
+ }
+
+private:
+
+ /// @brief Instance of the listener wrapped by this class.
+ HttpListenerPtr listener_;
+ /// @brief Instance of the response factory used by the listener.
+ TestHttpResponseCreatorFactoryPtr factory_;
+
+ /// @brief IPv4 leases to be used in the tests.
+ std::vector<Lease4Ptr> leases4_;
+
+ /// @brief Boolean flag indicating if the partner is running.
+ bool running_;
+
+ /// @brief Static date-time value to be returned.
+ std::string static_date_time_;
+
+ /// @brief Static scopes to be reported.
+ std::set<std::string> static_scopes_;
+
+ /// @brief Static count of lease updates not sent by the partner
+ /// because the other server was unavailable.
+ int64_t static_unsent_update_count_;
+};
+
+/// @brief Shared pointer to a partner.
+typedef boost::shared_ptr<HAPartner> HAPartnerPtr;
+
+/// @brief Test fixture class for the HA service state machine.
+class HAServiceStateMachineTest : public HAServiceTest {
+public:
+ /// @brief Constructor.
+ HAServiceStateMachineTest()
+ : HAServiceTest(), state_(),
+ partner_(new HAPartner(listener2_, factory2_)) {
+ }
+
+ /// @brief Creates common HA service instance from the provided configuration.
+ ///
+ /// The custom @c state_ object is created and it replaces the default
+ /// @c state_ object of the HA service.
+ ///
+ /// @param config pointer to the configuration to be used by the service.
+ /// @param server_type server type, i.e. DHCPv4 or DHCPv6.
+ void startService(const HAConfigPtr& config,
+ const HAServerType& server_type = HAServerType::DHCPv4) {
+ config->setHeartbeatDelay(1);
+ config->setSyncPageLimit(1000);
+ createSTService(network_state_, config, server_type);
+ // Replace default communication state with custom state which exposes
+ // protected members and methods.
+ state_.reset(new NakedCommunicationState4(io_service_, config));
+ service_->communication_state_ = state_;
+ // Move the state machine from initial state to "waiting" state.
+ service_->runModel(HAService::NOP_EVT);
+ }
+
+ /// @brief Runs IO service until specified event occurs.
+ ///
+ /// This method runs IO service until state machine is run as a result
+ /// of receiving a response to an IO operation. IO operations such as
+ /// lease updates, heartbeats etc. trigger state machine changes.
+ /// We can capture certain events to detect when a response to the heartbeat
+ /// or other control commands is received. This is useful to return control
+ /// to a test to verify that the state machine remains in the expected state
+ /// after receiving such response.
+ ///
+ /// @param event an event which should trigger IO service to stop.
+ void waitForEvent(const int event) {
+ ASSERT_NE(event, HAService::NOP_EVT);
+
+ service_->postNextEvent(HAService::NOP_EVT);
+
+ // Run IO service until the event occurs.
+ runIOService(TEST_TIMEOUT, [this, event]() {
+ return (service_->getLastEvent() == event);
+ });
+
+ service_->postNextEvent(HAService::NOP_EVT);
+ }
+
+ /// @brief Convenience method checking if HA service is currently running
+ /// recurring heartbeat.
+ ///
+ /// @return true if the heartbeat is run.
+ bool isDoingHeartbeat() {
+ return (state_->isHeartbeatRunning());
+ }
+
+ /// @brief Convenience method checking if HA service has detected communications
+ /// interrupted condition.
+ ///
+ /// @return true if the communications interrupted condition deemed, false
+ /// otherwise.
+ bool isCommunicationInterrupted() {
+ return (state_->isCommunicationInterrupted());
+ }
+
+ /// @brief Convenience method checking if communication failure has been
+ /// detected by the HA service based on the analysis of the DHCP traffic.
+ ///
+ /// @return true if the communication failure is deemed, false otherwise.
+ bool isFailureDetected() {
+ return (state_->failureDetected());
+ }
+
+ /// @brief Simulates a case when communication with the partner has failed
+ /// for a time long enough to assume communications interrupted condition.
+ ///
+ /// This case is simulated by modifying the last poking time far into the
+ /// past.
+ void simulateNoCommunication() {
+ state_->modifyPokeTime(-1000);
+ }
+
+ /// @brief Simulates reception of unanswered DHCP queries by the partner.
+ ///
+ /// This case is simulated by creating a large number of queries with
+ /// secs field set to high value.
+ void simulateDroppedQueries() {
+ // Create 100 packets. Around 50% of them should be assigned to the
+ // partner if load balancing is performed.
+ const unsigned queries_num = 100;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // Set large secs field value.
+ query4->setSecs(0x00EF);
+ // This function, besides checking if the query is in scope,
+ // updates unanswered message counters. If the counters exceed
+ // a configured value the communication failure is assumed.
+ static_cast<void>(service_->inScope(query4));
+ }
+ // The state machine needs to react to the dropped queries. Therefore
+ // we run the machine now.
+ service_->runModel(HAService::NOP_EVT);
+ }
+
+ /// @brief Simulates rejected lease updates by the partner.
+ ///
+ /// Too many rejected lease updates should transition the server to the
+ /// terminated state.
+ /// @param leases_num number of simulated rejected leases.
+ void simulateRejectedLeaseUpdates(const unsigned leases_num = 100) {
+ // If the state machine is about to transition to another state,
+ // let's make sure it performs this transition.
+ service_->runModel(HAService::NOP_EVT);
+ // Simulate the rejected lease updates.
+ for (auto i = 0; i < leases_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ static_cast<void>(state_->reportRejectedLeaseUpdate(query4, 100));
+ }
+ // The state machine needs to react to the rejected leases.
+ service_->runModel(HAService::NOP_EVT);
+ }
+
+ /// @brief Checks transitions dependent on the partner's state.
+ ///
+ /// This method uses @c partner_ object to control the state of the partner.
+ /// This method must not be used to test transitions from the syncing state
+ /// because this state includes synchronous IO operations. There is a
+ /// separate test for the transitions from the syncing state.
+ ///
+ /// @param my_state initial state of this server.
+ /// @param partner_state state of the partner.
+ /// @param final_state expected state to transition to.
+ void testTransition(const MyState& my_state, const PartnerState& partner_state,
+ const FinalState& final_state) {
+ // We need to shutdown the partner only if the partner is to be in the
+ // 'unavailable state'.
+ if (partner_state.state_ != HA_UNAVAILABLE_ST) {
+ // This function is not meant for testing transitions from the syncing
+ // state when partner is available.
+ ASSERT_NE(my_state.state_, HA_SYNCING_ST);
+ partner_->setControlResult(CONTROL_RESULT_SUCCESS);
+
+ } else {
+ partner_->setControlResult(CONTROL_RESULT_ERROR);
+ }
+
+ // Transition this server to the desired initial state.
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+ // Transition the partner to the desired state.
+ partner_->transition(service_->getStateLabel(partner_state.state_));
+ // Run the heartbeat.
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ // Make sure that this server ended up in the expected state.
+ EXPECT_EQ(final_state.state_, service_->getCurrState())
+ << "expected transition to the '"
+ << service_->getStateLabel(final_state.state_)
+ << "' state for the partner state '" << service_->getStateLabel(partner_state.state_)
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+
+ // If the partner is unavailable we also have to verify the case when
+ // we detect that the partner is considered offline (after running the
+ // whole failure detection algorithm). The server in the in-maintenance
+ // state is excluded from this because it must not transition out of this
+ // state until an administrator makes some action.
+ if ((my_state.state_ != HA_IN_MAINTENANCE_ST) &&
+ (partner_state.state_ == HA_UNAVAILABLE_ST)) {
+ // Transition this server back to the initial state.
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+ // Simulate lack of communication between the servers.
+ simulateNoCommunication();
+ // Send the heartbeat again.
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+
+ // The load balancing server or the standby server is monitoring the stream
+ // of packets directed to the partner to detect delays in responses. The
+ // primary server in the hot standby configuration doesn't do it, because
+ // the partner is not meant to process any queries until it detects that
+ // the primary server is down. This is only done in states in which the
+ // DHCP service is enabled. Otherwise, the server doesn't receive DHCP
+ // queries it could analyze.
+ if (service_->network_state_->isServiceEnabled() &&
+ ((service_->config_->getHAMode() == HAConfig::LOAD_BALANCING) ||
+ service_->config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY)) {
+ // The server should remain in its current state until we also detect
+ // that the partner is not answering the queries.
+ ASSERT_EQ(final_state.state_, service_->getCurrState())
+ << "expected transition to the '"
+ << service_->getStateLabel(final_state.state_)
+ << "' state for the partner state '" << service_->getStateLabel(partner_state.state_)
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+
+ // Back to the original state again.
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+ // This time simulate no responses from the partner to the DHCP clients'
+ // requests. This should cause the server to transition to the partner
+ // down state regardless of the initial state.
+ simulateDroppedQueries();
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState())
+ << "expected transition to the 'partner-down' state, but transitioned"
+ " to the '" << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+
+ // The primary server in the hot-standby configuration should transition to
+ // the partner-down state when there is no communication with the partner
+ // over the control channel.
+ } else {
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState())
+ << "expected transition to the 'partner-down' state, but transitioned"
+ " to the '" << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+ }
+ }
+ }
+
+ /// @brief Checks transitions from the syncing state.
+ ///
+ /// This method uses @c partner_ object to control the state of the partner.
+ ///
+ /// @param final_state expected final server state.
+ void testSyncingTransition(const FinalState& final_state) {
+ // Transition to the syncing state.
+ service_->transition(HA_SYNCING_ST, HAService::NOP_EVT);
+ partner_->transition("ready");
+ state_->stopHeartbeat();
+
+ testSynchronousCommands([this]() {
+ service_->runModel(HAService::NOP_EVT);
+ });
+
+ state_->stopHeartbeat();
+
+ EXPECT_EQ(final_state.state_, service_->getCurrState())
+ << "expected transition to the '"
+ << service_->getStateLabel(final_state.state_)
+ << "' state, but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+ }
+
+ /// @brief Tests transition from any state to "terminated".
+ ///
+ /// @param my_state initial server state.
+ void testTerminateTransition(const MyState& my_state) {
+ // Set the partner's time way in the past so as the clock skew gets high.
+ partner_->setDateTime("Sun, 06 Nov 1994 08:49:37 GMT");
+ partner_->transition("ready");
+ // Transition this server to the desired initial state.
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+ // Run the heartbeat.
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ // The server should get into terminated state because of the high
+ // clock skew.
+ EXPECT_EQ(HA_TERMINATED_ST, service_->getCurrState())
+ << "expected transition to the 'terminated' state"
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+ }
+
+ /// @brief Tests transition between states when passive-backup mode is
+ /// in use.
+ ///
+ /// @param my_state initial server state.
+ /// @param final_state expected final state.
+ void testPassiveBackupTransition(const MyState& my_state,
+ const FinalState& final_state) {
+ service_->transition(my_state.state_, HAService::NOP_EVT);
+ service_->runModel(TestHAService::NOP_EVT);
+
+ EXPECT_EQ(final_state.state_, service_->getCurrState())
+ << "expected transition to the '"
+ << service_->getStateLabel(final_state.state_)
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+ }
+
+ /// @brief Test that the server is serving expected scopes while being in a
+ /// certain state.
+ ///
+ /// @param my_state state of the server.
+ /// @param scopes vector of scopes which the server is expected to handle in this
+ /// state.
+ /// @param dhcp_enabled Indicates whether DHCP service is expected to be enabled
+ /// or disabled in the given state.
+ /// @param event Event to be passed to the tested handler.
+ void expectScopes(const MyState& my_state, const std::vector<std::string>& scopes,
+ const bool dhcp_enabled, const int event = TestHAService::NOP_EVT) {
+
+ // If expecting no scopes, let's enable some scope to make sure that the
+ // code changes this setting.
+ if (scopes.empty()) {
+ service_->query_filter_.serveScope("server1");
+
+ } else {
+ // If expecting some scopes, let's initially clear the scopes to make
+ // sure that the code sets it.
+ service_->query_filter_.serveNoScopes();
+ }
+
+ // Also, let's preset the DHCP server state to the opposite of the expected
+ // state.
+ if (dhcp_enabled) {
+ service_->network_state_->disableService(NetworkState::Origin::HA_COMMAND);
+
+ } else {
+ service_->network_state_->enableService(NetworkState::Origin::HA_COMMAND);
+ }
+
+ // Transition to the desired state.
+ service_->postNextEvent(event);
+ service_->verboseTransition(my_state.state_);
+ // Run the handler.
+ service_->runModel(TestHAService::NOP_EVT);
+ // First, check that the number of handlded scope is equal to the number of
+ // scopes specified as an argument.
+ ASSERT_EQ(scopes.size(), service_->query_filter_.getServedScopes().size())
+ << "test failed for state '" << service_->getStateLabel(my_state.state_)
+ << "'";
+
+ // Now, verify that each specified scope is handled.
+ for(auto scope : scopes) {
+ EXPECT_TRUE(service_->query_filter_.amServingScope(scope))
+ << "test failed for state '" << service_->getStateLabel(my_state.state_)
+ << "'";
+ }
+ // Verify if the DHCP service is enabled or disabled.
+ EXPECT_EQ(dhcp_enabled, service_->network_state_->isServiceEnabled())
+ << "test failed for state '" << service_->getStateLabel(my_state.state_)
+ << "'";
+ }
+
+ /// @brief Transitions the server to the specified state and checks if the
+ /// HA service would send lease updates in this state.
+ ///
+ /// @param my_state this server's state
+ /// @param peer_config configuration of the server to which lease updates are
+ /// to be sent.
+ /// @return true if the lease updates would be sent, false otherwise.
+ bool expectLeaseUpdates(const MyState& my_state,
+ const HAConfig::PeerConfigPtr& peer_config) {
+ service_->verboseTransition(my_state.state_);
+ return (service_->shouldSendLeaseUpdates(peer_config));
+ }
+
+ /// @brief Transitions the server to the specified state and checks if the
+ /// HA service would queue lease updates in this state.
+ ///
+ /// @param my_state this server's state
+ /// @param peer_config configuration of the server to which lease updates are
+ /// to be sent or queued.
+ /// @return true if the lease updates would be queued, false otherwise.
+ bool expectQueueLeaseUpdates(const MyState& my_state,
+ const HAConfig::PeerConfigPtr& peer_config) {
+ service_->verboseTransition(my_state.state_);
+ return (service_->shouldQueueLeaseUpdates(peer_config));
+ }
+
+ /// @brief Transitions the server to the specified state and checks if the
+ /// HA service is sending heartbeat in this state.
+ ///
+ /// @param my_state this server's state
+ /// @return true if the heartbeat is sent in this state, false otherwise.
+ bool expectHeartbeat(const MyState& my_state) {
+ service_->verboseTransition(my_state.state_);
+ service_->runModel(TestHAService::NOP_EVT);
+ return (isDoingHeartbeat());
+ }
+
+ /// @brief Transitions the server to the specified state and checks that it
+ /// clears the rejected leases counter.
+ ///
+ /// @param my_state this server's state
+ /// @return true if the rejected leases counter has been cleared.
+ bool expectClearRejectedLeases(const MyState& my_state) {
+ simulateRejectedLeaseUpdates(1);
+ service_->verboseTransition(my_state.state_);
+ service_->runModel(TestHAService::NOP_EVT);
+ return (state_->getRejectedLeaseUpdatesCount() == 0);
+ }
+
+ /// @brief Pointer to the communication state used in the tests.
+ NakedCommunicationState4Ptr state_;
+ /// @brief Pointer to the partner used in some tests.
+ HAPartnerPtr partner_;
+};
+
+
+// Test the following scenario:
+// 1. I show up in waiting state and look around
+// 2. My partner doesn't respond over control channel
+// 3. I start analyzing partner's packets and see that
+// it doesn't respond.
+// 4. I transition to partner down state.
+// 5. Partner finally shows up and eventually transitions to the ready state.
+// 6. I see the partner being ready, so I synchronize myself and eventually
+// transition to the load-balancing state.
+// 7. Next, the partner crashes again.
+// 8. I detect partner's crash and transition back to partner down.
+// 9. While being in the partner down state, I find that the partner
+// is available and it is doing load balancing.
+// 10. I stay in the partner-down state to force the partner to transition
+// to the waiting state and synchronize its database.
+TEST_F(HAServiceStateMachineTest, waitingParterDownLoadBalancingPartnerDown) {
+ HAConfigPtr config_storage = createValidConfiguration();
+ // Disable syncing leases to avoid transitions via the syncing state.
+ // In this state it is necessary to perform synchronous tasks.
+ config_storage->setSyncLeases(false);
+ startService(config_storage);
+
+ // Start the server: offline ---> WAITING state.
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: no heartbeat response for a long period of time.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ simulateNoCommunication();
+ ASSERT_TRUE(isDoingHeartbeat());
+
+ // WAITING state: communication interrupted. In this state we don't analyze
+ // packets ('secs' field) because the DHCP service is disabled.
+ // WAITING ---> PARTNER DOWN
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+
+ // PARTNER DOWN state: still no response from the partner.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+
+ // Partner shows up and (eventually) transitions to READY state.
+ HAPartner partner(listener2_, factory2_);
+ partner.setScopes({ "server1", "server2" });
+ partner.setUnsentUpdateCount(10);
+ partner.transition("ready");
+ partner.startup();
+
+ // PARTNER DOWN state: receive a response from the partner indicating that
+ // the partner is in READY state.
+ // PARTNER DOWN ---> WAITING
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // WAITING state: synchronization is disabled to simplify this test. The
+ // server should transition straight to the ready state.
+ // WAITING --> READY
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // READY --> LOAD BALANCING
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // Ensure that the state handler is invoked.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+
+ // Check the reported info about servers.
+ ConstElementPtr ha_servers = service_->processStatusGet();
+ ASSERT_TRUE(ha_servers);
+
+ // Hard to know what is the age of the remote data. Therefore, let's simply
+ // grab it from the response.
+ ASSERT_EQ(Element::map, ha_servers->getType());
+ auto remote = ha_servers->get("remote");
+ ASSERT_TRUE(remote);
+ EXPECT_EQ(Element::map, remote->getType());
+ auto age = remote->get("age");
+ ASSERT_TRUE(age);
+ EXPECT_EQ(Element::integer, age->getType());
+ auto age_value = age->intValue();
+ EXPECT_GE(age_value, 0);
+
+ // Now append it to the whole structure for comparison.
+ std::ostringstream s;
+ s << age_value;
+
+ std::string expected = "{"
+ " \"local\": {"
+ " \"role\": \"primary\","
+ " \"scopes\": [ \"server1\" ], "
+ " \"state\": \"load-balancing\""
+ " }, "
+ " \"remote\": {"
+ " \"age\": " + s.str() + ","
+ " \"in-touch\": true,"
+ " \"role\": \"secondary\","
+ " \"last-scopes\": [ \"server1\", \"server2\" ],"
+ " \"last-state\": \"ready\","
+ " \"communication-interrupted\": false,"
+ " \"connecting-clients\": 0,"
+ " \"unacked-clients\": 0,"
+ " \"unacked-clients-left\": 0,"
+ " \"analyzed-packets\": 0"
+ " }"
+ "}";
+ EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers));
+
+ // Crash the partner and see whether our server can return to the partner
+ // down state.
+ partner.setControlResult(CONTROL_RESULT_ERROR);
+
+ // LOAD BALANCING state: wait for the next heartbeat to occur. This heartbeat
+ // fails causing the server to enter communication-recovery state in which the
+ // server is collecting lease updates to be sent when the communication is
+ // resumed.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // COMMUNICATION RECOVERY state: simulate lack of communication for a longer
+ // period of time. We should remain the communication-recovery state and
+ // keep analyzing the traffic directed to partner.
+ simulateNoCommunication();
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_TRUE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // COMMUNICATION RECOVERY state: simulate a lot of unanswered DHCP
+ // messages to the partner. This server should detect that the partner is
+ // not answering and transition to partner down state.
+ // COMMUNICATION RECOVERY ---> PARTNER DOWN
+ simulateDroppedQueries();
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_TRUE(isCommunicationInterrupted());
+ ASSERT_TRUE(isFailureDetected());
+
+ // Start the partner again and transition it to the load balancing state.
+ partner.setControlResult(CONTROL_RESULT_SUCCESS);
+ partner.transition("load-balancing");
+
+ // PARTNER DOWN state: the partner shows up in the load-balancing state.
+ // It may happen when the partner did not crash but there was a temporary
+ // communication error with it. It is possible that this server was not
+ // configured to monitor unacked clients and that's why it transitioned
+ // to the partner-down state. The partner may be configured differently.
+ // The partner was not receiving lease updates from us, so we need to
+ // force it to transition to the waiting state and synchronize. We stay
+ // in the partner-down state as long as necessary to force the partner
+ // to synchronize.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+}
+
+// Test the following scenario:
+// 1. I show up in waiting state and look around.
+// 2. My partner is offline and is not responding over the control channel.
+// 3. I can't communicate with the partner so I transition to the partner-down
+// state.
+// 4. Partner shows up and eventually transitions to the ready state.
+// 5. I see the partner being ready, so I synchronize myself and eventually
+// transition to the load-balancing state.
+// 6. Partner stops responding again.
+// 7. I monitor communication with the partner and eventually consider the
+// communication to be interrupted.
+// 8. I start monitoring the DHCP traffic directed to the partner and observe
+// delays in responses.
+// 9. I transition to the partner-down state again seeing that the certain
+// number of clients can't communicate with the partner.
+// 10. The partner unexpectedly shows up in the hot-standby mode. I stay in
+// the partner-down state to force the partner to transition to the waiting
+// state and synchronize its database.
+TEST_F(HAServiceStateMachineTest, waitingParterDownHotStandbyPartnerDown) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ // Disable syncing leases to avoid transitions via the syncing state.
+ // In this state it is necessary to perform synchronous tasks.
+ valid_config->setSyncLeases(false);
+
+ // Start the server: offline ---> WAITING state.
+ startService(valid_config);
+
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: no heartbeat response for a long period of time.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ simulateNoCommunication();
+ ASSERT_TRUE(isDoingHeartbeat());
+
+ // WAITING state: communication interrupted. In this state we don't analyze
+ // packets ('secs' field) because the DHCP service is disabled.
+ // WAITING ---> PARTNER DOWN
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+
+ // PARTNER DOWN state: still no response from the partner.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+
+ // Partner shows up and (eventually) transitions to READY state.
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->setUnsentUpdateCount(10);
+ partner_->transition("ready");
+ partner_->startup();
+
+ // PARTNER DOWN state: receive a response from the partner indicating that
+ // the partner is in READY state.
+ // PARTNER DOWN ---> WAITING
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // WAITING state: synchronization is disabled to simplify this test. The
+ // server should transition straight to the ready state.
+ // WAITING --> READY
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // Primary server must transition to the hot-standby state first before
+ // standby can transition.
+ partner_->transition("hot-standby");
+
+ // READY --> HOT STANDBY
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // Crash the partner and see whether our server can return to the partner
+ // down state.
+ partner_->setControlResult(CONTROL_RESULT_ERROR);
+
+ // HOT STANDBY state: wait for the next heartbeat to occur and make
+ // sure that a single heartbeat loss is not yet causing us to assume
+ // partner down condition.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // HOT STANDBY state: simulate lack of communication for a longer
+ // period of time. We should remain in the hot-standby state waiting for
+ // unanswered DHCP traffic before going to partner-down state.
+ simulateNoCommunication();
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_HOT_STANDBY_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_TRUE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+
+ // HOT STANDBY state: simulate a lot of unanswered DHCP
+ // messages to the partner. This server should detect that the partner is
+ // not answering and transition to partner down state.
+ // HOT STANDBY ---> PARTNER DOWN
+ simulateDroppedQueries();
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_TRUE(isCommunicationInterrupted());
+ ASSERT_TRUE(isFailureDetected());
+
+ // Start the partner again and transition it to the hot standby state.
+ partner_->setControlResult(CONTROL_RESULT_SUCCESS);
+ partner_->transition("hot-standby");
+
+ // PARTNER DOWN state: the partner shows up in the hot-standby state.
+ // It may happen when the partner did not crash but there was a temporary
+ // communication error with it. The partner was not receiving lease updates
+ // from us, so we need to force it to transition to the waiting state and
+ // synchronize. We stay in the partner-down state as long as necessary to
+ // force the partner to synchronize.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
+ ASSERT_TRUE(isDoingHeartbeat());
+ ASSERT_FALSE(isCommunicationInterrupted());
+ ASSERT_FALSE(isFailureDetected());
+}
+
+// Test the following scenario:
+// 1. I begin in a load-balancing state.
+// 2. My partner is offline.
+// 3. I transition to the communication-recovery state.
+// 4. My partner shows up in the load-balancing state.
+// 5. I see the partner so I get back to load-balancing state as well.
+TEST_F(HAServiceStateMachineTest, loadBalancingCommRecoveryLoadBalancing) {
+ startService(createValidConfiguration());
+ service_->verboseTransition(HA_LOAD_BALANCING_ST);
+ service_->runModel(HAService::NOP_EVT);
+
+ // Simulate that the partner is not responding to my queries.
+ simulateNoCommunication();
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+
+ HAPartner partner(listener2_, factory2_, "load-balancing");
+ partner.startup();
+
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+}
+
+// Test the following scenario:
+// 1. I show up in the waiting state.
+// 2. My partner appears to be in the partner-down state.
+// 3. I proceed to the syncing state to fetch leases from the partner.
+// 4. The first attempt to fetch leases is unsuccessful.
+// 5. I remain in the syncing state until I am finally successful.
+// 6. I proceed to the ready state.
+// 7. I see that the partner is still in partner-down state, so I
+// wait for the partner to transition to load-balancing state.
+TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancing) {
+ // Partner is present and is in the PARTNER DOWN state.
+ HAPartner partner(listener2_, factory2_, "partner-down");
+ partner.startup();
+
+ // Start the server: offline ---> WAITING state.
+ startService(createValidConfiguration());
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: receive a response from the partner indicating that the
+ // partner is in the load balancing state. I should transition to the
+ // SYNCING state to fetch leases from the partner.
+ // WAITING ---> SYNCING state
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState());
+
+ // We better stop the heartbeat to not interfere with the synchronous
+ // commands.
+ state_->stopHeartbeat();
+
+ // The database synchronization is synchronous operation so we need to run
+ // the partner's IO service in thread (in background).
+ testSynchronousCommands([this, &partner]() {
+
+ // SYNCING state: the partner is up but it won't respond to the lease4-get-page
+ // command correctly. This should leave us in the SYNCING state until we finally
+ // can synchronize.
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState());
+
+ // We better stop the heartbeat to not interfere with the synchronous
+ // commands.
+ state_->stopHeartbeat();
+
+ // Enable the partner to correctly respond to the lease fetching and retry.
+ // We should successfully update the database and transition.
+ // SYNCING ---> READY
+ partner.enableRespondLeaseFetching();
+
+ // After previous attempt to synchronize the recorded partner state became
+ // "unavailable". This server won't synchronize until the heartbeat is
+ // sent which would indicate that the server is running. Therefore, we
+ // manually set the state of the partner to "partner-down".
+ state_->setPartnerState("partner-down");
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+ });
+
+ // READY state: I do another heartbeat but my partner still seems to be in the
+ // partner down state, so I can't transition to load balancing just yet. I
+ // still remain the READY state.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+
+ // READY state: partner transitions to the LOAD BALANCING state seeing that I
+ // am ready. I should see this transition and also transition to that state.
+ // READY ---> LOAD BALANCING.
+ partner.transition("load-balancing");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+}
+
+// Test the following scenario:
+// 1. I am a primary server which is started at the same time as secondary.
+// 2. I determine that we're the primary server so I go ahead and start
+// synchronizing.
+// 3. Partner should also determine that it is a secondary server so it should
+// remain in the waiting state until we're ready.
+// 4. I synchronize the database and transition to the ready state.
+// 5. The partner should see that I am ready and should start the transition.
+// 6. I remain ready until the partner gets to the ready state.
+// 7. I transition to the load balancing state when the partner is ready.
+// 8. The partner transitions to the load balancing state which doesn't
+// affect my state.
+TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancingPrimary) {
+ // Partner is present and is in the WAITING state.
+ HAPartner partner(listener2_, factory2_);
+ partner.enableRespondLeaseFetching();
+ partner.startup();
+
+ // Start the server: offline ---> WAITING state.
+ startService(createValidConfiguration());
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: both servers are in this state but our server is primary,
+ // so it transitions to the syncing state. The peer remains in the WAITING
+ // state until we're ready.
+ // WAITING --->SYNCING
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState());
+
+ // We better stop the heartbeat to not interfere with the synchronous
+ // commands.
+ state_->stopHeartbeat();
+
+ // SYNCING state: this server will synchronously fetch leases from the peer.
+ // Therefore, we need to run the IO service in the thread to allow for
+ // synchronous operations to complete. Once the leases are fetched it should
+ // transition to the READY state.
+ // SYNCING ---> READY.
+ testSynchronousCommands([this]() {
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+ });
+
+ // READY state: our partner sees that we're ready so it will start to
+ // synchronize. We remain in the READY state as long as the partner is not
+ // ready.
+ partner.transition("syncing");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+
+ // READY state: our partner appears to be ready. We can now start load
+ // balancing.
+ // READY ---> LOAD BALANCING.
+ partner.transition("ready");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+
+ // LOAD BALANCING state: our partner should eventually transition to the
+ // LOAD BALANCING state. This should not affect us doing load balancing.
+ partner.transition("load-balancing");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+}
+
+// Test the following scenario:
+// 1. I am secondary server configured for load balancing and started at the
+// same time as the primary server.
+// 2. I determine that I am a secondary server, so I remain in the waiting
+// state until the primary server indicates that it is ready.
+// 3. I start synchronizing my lease database when the partner is ready.
+// 4. I transition to the ready state and leave in that state until I see
+// the primary transition to the load balancing state.
+// 5. I also transition to the load balancing state at that point.
+TEST_F(HAServiceStateMachineTest, waitingSyncingReadyLoadBalancingSecondary) {
+ // Partner is present and is in the WAITING state.
+ HAPartner partner(listener_, factory_);
+ partner.enableRespondLeaseFetching();
+ partner.startup();
+
+ // Create the configuration in which we're the secondary server doing
+ // load balancing.
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setThisServerName("server2");
+
+ // Start the server: offline ---> WAITING state.
+ startService(valid_config);
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: our partner is a primary so we remain in the WAITING state
+ // until it indicates it is ready.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: the partner is synchronizing the lease database. We still
+ // remain in the WAITING state.
+ partner.transition("syncing");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState());
+
+ // WAITING state: partner transitions to the READY state. This is a signal
+ // for us to start synchronization of the lease database.
+ // WAITING ---> SYNCING.
+ partner.transition("ready");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_SYNCING_ST, service_->getCurrState());
+
+ // We better stop the heartbeat to not interfere with the synchronous
+ // commands.
+ state_->stopHeartbeat();
+
+ // SYNCING state: this server will synchronously fetch leases from the peer.
+ // Therefore, we need to run the IO service in the thread to allow for
+ // synchronous operations to complete. Once the leases are fetched it should
+ // transition to the READY state.
+ // SYNCING ---> READY.
+ testSynchronousCommands([this]() {
+ service_->runModel(HAService::NOP_EVT);
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+ });
+
+ // READY state: we remain ready until the other server transitions to the
+ // LOAD BALANCING state.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_READY_ST, service_->getCurrState());
+
+ // READY state: our primary server transitions to the LOAD BALANCING state.
+ // We can now also transition to this state.
+ // READY ---> LOAD BALANCING.
+ partner.transition("load-balancing");
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_LOAD_BALANCING_ST, service_->getCurrState());
+}
+
+// This test checks all combinations of server and partner states and the
+// resulting state to which the server transitions. This server is primary.
+// There is another test which validates state transitions from the
+// secondary server perspective.
+TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
+ partner_->startup();
+
+ startService(createValidConfiguration());
+
+ // COMMUNICATION RECOVERY state transitions
+ {
+ SCOPED_TRACE("COMMUNICATION RECOVERY state transitions");
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+ }
+
+ // LOAD BALANCING state transitions
+ {
+ SCOPED_TRACE("LOAD BALANCING state transitions");
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+ }
+
+ // in-maintenance state transitions
+ {
+ SCOPED_TRACE("in-maintenance state transitions");
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+ }
+
+ // PARTNER DOWN state transitions
+ {
+ SCOPED_TRACE("PARTNER DOWN state transitions");
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // PARTNER in-maintenance state transitions
+ {
+ SCOPED_TRACE("PARTNER in-maintenance state transitions");
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // READY state transitions
+ {
+ SCOPED_TRACE("READY state transitions");
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_READY_ST));
+ }
+
+ // WAITING state transitions
+ {
+ SCOPED_TRACE("WAITING state transitions");
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_WAITING_ST));
+ }
+}
+
+// This test checks that the server in the load balancing mode does not
+// transition to the "syncing" state when "sync-leases" is disabled.
+TEST_F(HAServiceStateMachineTest, noSyncingTransitionsLoadBalancingPrimary) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setSyncLeases(false);
+ startService(valid_config);
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+}
+
+// This test verifies that the load balancing server does not transition to
+// the communication recovery state when delayed-updates-limit is set
+// to 0.
+TEST_F(HAServiceStateMachineTest, noCommunicationRecoverytransitionsLoadBalancingPrimary) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setDelayedUpdatesLimit(0);
+ startService(valid_config);
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+}
+
+// This test checks that the server in the load balancing mode transitions to
+// the "terminated" state when the clock skew gets high.
+TEST_F(HAServiceStateMachineTest, terminateTransitionsLoadBalancingPrimary) {
+ partner_->startup();
+
+ startService(createValidConfiguration());
+
+ testTerminateTransition(MyState(HA_LOAD_BALANCING_ST));
+ testTerminateTransition(MyState(HA_PARTNER_DOWN_ST));
+ testTerminateTransition(MyState(HA_READY_ST));
+ testTerminateTransition(MyState(HA_WAITING_ST));
+}
+
+// This test checks that the server does not transition out of the waiting state
+// to the terminated state when the server is restarted but the clock skew has
+// been corrected.
+TEST_F(HAServiceStateMachineTest, terminateNoTransitionOnRestart) {
+ partner_->startup();
+ startService(createValidConfiguration());
+
+ // Set partner's time to the current time. This guarantees that the clock
+ // skew is below 60s and there is no reason for the server to transition
+ // to the terminated state.
+ partner_->setDateTime(HttpDateTime().rfc1123Format());
+ // The partner is in the terminated state to simulate sequential restart
+ // of the two servers from the terminated state.
+ partner_->transition("terminated");
+ // This server is in the waiting state which simulates the restart case.
+ service_->transition(HA_WAITING_ST, HAService::NOP_EVT);
+ // Run the heartbeat.
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ // The server should remain in the waiting state because the clock skew
+ // is low.
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState())
+ << "expected that the server remains in 'waiting' state"
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+
+ // Now, let's set the partner's time way to the past to verify that this
+ // server transitions to the 'terminated' state if the administrator
+ // failed to sync the clocks prior to the restart.
+ partner_->setDateTime("Sun, 06 Nov 1994 08:49:37 GMT");
+ // Run the heartbeat.
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+ EXPECT_EQ(HA_WAITING_ST, service_->getCurrState())
+ << "expected that the server transitions to the 'terminated' state"
+ << "', but transitioned to the '"
+ << service_->getStateLabel(service_->getCurrState())
+ << "' state";
+}
+
+// This test checks all combinations of server and partner states and the
+// resulting state to which the server transitions. This server is secondary.
+// There is another test which validates state transitions from the
+// primary server perspective.
+TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setThisServerName("server2");
+ startService(valid_config);
+
+ partner_->startup();
+
+ // COMMUNICATION RECOVERY state transitions
+ {
+ SCOPED_TRACE("COMMUNICATION RECOVERY state transitions");
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+
+ testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+ }
+
+ // LOAD BALANCING state transitions
+ {
+ SCOPED_TRACE("LOAD BALANCING state transitions");
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_COMMUNICATION_RECOVERY_ST));
+ }
+
+ // in-maintenance state transitions
+ {
+ SCOPED_TRACE("in-maintenance state transitions");
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+ }
+
+ // PARTNER DOWN state transitions
+ {
+ SCOPED_TRACE("PARTNER DOWN state transitions");
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // PARTNER in-maintenance state transitions
+ {
+ SCOPED_TRACE("PARTNER in-maintenance state transitions");
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // READY state transitions
+ {
+ SCOPED_TRACE("READY state transitions");
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_READY_ST));
+ }
+
+ // WAITING state transitions
+ {
+ SCOPED_TRACE("WAITING state transitions");
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_WAITING_ST));
+ }
+}
+
+// This test checks that the server in the load balancing mode does not
+// transition to the "syncing" state when "sync-leases" is disabled.
+// This is the secondary server case.
+TEST_F(HAServiceStateMachineTest, noSyncingTransitionsLoadBalancingSecondary) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setThisServerName("server2");
+ valid_config->setSyncLeases(false);
+ startService(valid_config);
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+}
+
+// This test checks that the secondary server in the load balancing mode
+// transitions to the "terminated" state when the clock skew gets high.
+TEST_F(HAServiceStateMachineTest, terminateTransitionsLoadBalancingSecondary) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setThisServerName("server2");
+ startService(valid_config);
+
+ testTerminateTransition(MyState(HA_LOAD_BALANCING_ST));
+ testTerminateTransition(MyState(HA_PARTNER_DOWN_ST));
+ testTerminateTransition(MyState(HA_READY_ST));
+ testTerminateTransition(MyState(HA_WAITING_ST));
+}
+
+// This test verifies that the backup server transitions to its own state.
+TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingBackup) {
+ HAConfigPtr valid_config = createValidConfiguration();
+
+ // server3 is marked as a backup server.
+ valid_config->setThisServerName("server3");
+ startService(valid_config);
+
+ // The server should transition to the backup state and stay there.
+ for (unsigned i = 0; i < 10; ++i) {
+ service_->runModel(HAService::NOP_EVT);
+ ASSERT_EQ(HA_BACKUP_ST, service_->getCurrState());
+ // In the backup state the DHCP service is disabled by default.
+ // It can only be enabled manually.
+ ASSERT_FALSE(service_->network_state_->isServiceEnabled());
+ ASSERT_EQ(0, service_->query_filter_.getServedScopes().size());
+ }
+}
+
+// This test verifies transitions from the syncing state in the load
+// balancing configuration.
+TEST_F(HAServiceStateMachineTest, syncingTransitionsLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ startService(valid_config);
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+
+ // The syncing state handler doesn't start synchronization until it
+ // detects that the partner is online. It may remember that from the
+ // previous heartbeat attempts. If the partner appears to be unavailable
+ // it will continue heartbeats before it synchronizes. This prevents the
+ // server from making endless attempts to synchronize without any chance
+ // to succeed. We verify that the server is not trying to synchronize
+ // by checking that the last event is not the one associated with the
+ // synchronization attempt.
+ ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT);
+ ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT);
+
+ // Run the syncing state handler.
+ testSyncingTransition(FinalState(HA_SYNCING_ST));
+
+ // We should see no synchronization attempts because the partner is
+ // offline.
+ EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT);
+ EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT);
+
+ // Startup the partner.
+ partner_->enableRespondLeaseFetching();
+ partner_->startup();
+
+ // We haven't been running heartbeats so we have to manually set the
+ // partner's state to something other than 'unavailable'.
+ state_->setPartnerState("ready");
+
+ // Retry the test.
+ testSyncingTransition(FinalState(HA_READY_ST));
+ // This time the server should have synchronized.
+ EXPECT_EQ(HAService::HA_SYNCING_SUCCEEDED_EVT, service_->getLastEvent());
+}
+
+// This test verifies that the HA state machine can be paused in certain states
+// when the server is operating in load balancing mode. The test also verifies
+// that heartbeat is active even if the state machine is paused.
+TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPause) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration();
+ auto state_machine = valid_config->getStateMachineConfig();
+
+ // Set state machine pausing in various states.
+ state_machine->getStateConfig(HA_LOAD_BALANCING_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_PARTNER_DOWN_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_READY_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_SYNCING_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_TERMINATED_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_WAITING_ST)->setPausing("always");
+
+ startService(valid_config);
+
+ {
+ SCOPED_TRACE("LOAD BALANCING state transitions");
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+
+ EXPECT_TRUE(service_->unpause());
+ // An additional attempt to unpause should return false.
+ EXPECT_FALSE(service_->unpause());
+
+ testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+ }
+
+ {
+ SCOPED_TRACE("PARTNER DOWN state transitions");
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+
+ EXPECT_TRUE(service_->unpause());
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+ }
+
+
+ {
+ SCOPED_TRACE("READY state transitions");
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_READY_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+
+ EXPECT_TRUE(service_->unpause());
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_LOAD_BALANCING_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+ }
+
+
+ {
+ SCOPED_TRACE("WAITING state transitions");
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+
+ EXPECT_TRUE(service_->unpause());
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_SYNCING_ST));
+ EXPECT_TRUE(state_->isHeartbeatRunning());
+ }
+}
+
+// This test verifies that the HA state machine can be paused in the syncing
+// state.
+TEST_F(HAServiceStateMachineTest, syncingTransitionsLoadBalancingPause) {
+ HAConfigPtr valid_config = createValidConfiguration();
+
+ auto state_machine = valid_config->getStateMachineConfig();
+
+ // Pause state machine in various states.
+ state_machine->getStateConfig(HA_LOAD_BALANCING_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_PARTNER_DOWN_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_READY_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_SYNCING_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_TERMINATED_ST)->setPausing("always");
+ state_machine->getStateConfig(HA_WAITING_ST)->setPausing("always");
+
+ startService(valid_config);
+ waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT);
+
+ // The syncing state handler doesn't start synchronization until it
+ // detects that the partner is online. It may remember that from the
+ // previous heartbeat attempts. If the partner appears to be unavailable
+ // it will continue heartbeats before it synchronizes. This prevents the
+ // server from making endless attempts to synchronize without any chance
+ // to succeed. We verify that the server is not trying to synchronize
+ // by checking that the last event is not the one associated with the
+ // synchronization attempt.
+ ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT);
+ ASSERT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT);
+
+ // Startup the partner.
+ partner_->enableRespondLeaseFetching();
+ partner_->startup();
+
+ // We haven't been running heartbeats so we have to manually set the
+ // partner's state to something other than 'unavailable'.
+ state_->setPartnerState("ready");
+
+ // Run the syncing state handler.
+ testSyncingTransition(FinalState(HA_SYNCING_ST));
+
+ // We should see no synchronization attempts because the server is paused
+ // in this state.
+ EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_FAILED_EVT);
+ EXPECT_NE(service_->getLastEvent(), HAService::HA_SYNCING_SUCCEEDED_EVT);
+
+ // Unpause the state machine.
+ EXPECT_TRUE(service_->unpause());
+
+ // Retry the test. It should now transition to the ready state.
+ testSyncingTransition(FinalState(HA_READY_ST));
+
+ // This time the server should have synchronized.
+ EXPECT_EQ(HAService::HA_SYNCING_SUCCEEDED_EVT, service_->getLastEvent());
+}
+
+// This test verifies that the server takes ownership of the given scopes
+// and whether the DHCP service is disabled or enabled in certain states.
+TEST_F(HAServiceStateMachineTest, scopesServingLoadBalancing) {
+ startService(createValidConfiguration());
+
+ // LOAD BALANCING and TERMINATED: serving my own scope.
+ expectScopes(MyState(HA_LOAD_BALANCING_ST), { "server1" }, true);
+ expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true);
+
+ // PARTNER DOWN and PARTNER IN MAINTENANCE: serving both scopes.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1", "server2" }, true);
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1", "server2" }, true);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies that the server does not take ownership of the
+// partner's scope when auto-failover parameter is set to false.
+TEST_F(HAServiceStateMachineTest, scopesServingLoadBalancingNoFailover) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->getThisServerConfig()->setAutoFailover(false);
+ startService(valid_config);
+
+ // LOAD BALANCING and TERMINATED: serving my own scope.
+ expectScopes(MyState(HA_LOAD_BALANCING_ST), { "server1" }, true);
+ expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true);
+
+ // PARTNER DOWN: still serving my own scope because auto-failover is disabled.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true);
+
+ // PARTNER IN MAINTENANCE: always serving all scopes.
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1", "server2" }, true);
+
+ // Same for the partner-down case during maintenance.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1", "server2" }, true,
+ HAService::HA_MAINTENANCE_START_EVT);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies if the server would send lease updates to the partner
+// while being in various states. The HA configuration is load balancing.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config));
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// Check that lease updates are sent to the backup server even when the
+// secondary is in the partner-down state.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesToBackup) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setWaitBackupAck(false);
+ startService(valid_config);
+
+ // Send the updates to the backup server.
+ HAConfig::PeerConfigPtr backup_config = valid_config->getPeerConfig("server3");
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), backup_config));
+}
+
+// This test verifies if the server would not send lease updates to the
+// partner if lease updates are administratively disabled.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesDisabledLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setSendLeaseUpdates(false);
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// Check that lease updates to the backup server are not queued.
+TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesToBackup) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setWaitBackupAck(false);
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr backup_config = valid_config->getPeerConfig("server3");
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), backup_config));
+}
+
+// This test verifies if the server would queue lease updates to the partner
+// while being in communication-recovery state. The HA configuration is load
+// balancing.
+TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_TRUE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// This test verifies if the server would not queue lease updates to the
+// partner if lease updates are administratively disabled.
+TEST_F(HAServiceStateMachineTest, shouldQueueLeaseUpdatesDisabledLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ valid_config->setSendLeaseUpdates(false);
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_COMMUNICATION_RECOVERY_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_LOAD_BALANCING_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectQueueLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// This test verifies if the server would send heartbeat to the partner
+// while being in various states. The HA configuration is load balancing.
+TEST_F(HAServiceStateMachineTest, heartbeatLoadBalancing) {
+ HAConfigPtr valid_config = createValidConfiguration();
+ startService(valid_config);
+
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_LOAD_BALANCING_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_DOWN_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_READY_ST)));
+ EXPECT_FALSE(expectHeartbeat(MyState(HA_TERMINATED_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_WAITING_ST)));
+}
+
+// This test checks all combinations of server and partner states and the
+// resulting state to which the server transitions. This server is primary.
+// There is another test which validates state transitions from the
+// standby server perspective.
+TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ // HOT STANDBY state transitions
+ {
+ SCOPED_TRACE("HOT STANDBY state transitions");
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+ }
+
+ // in-maintenance state transitions
+ {
+ SCOPED_TRACE("in-maintenance state transitions");
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+ }
+
+ // PARTNER DOWN state transitions
+ {
+ SCOPED_TRACE("PARTNER DOWN state transitions");
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // PARTNER in-maintenance state transitions
+ {
+ SCOPED_TRACE("PARTNER in-maintenance state transitions");
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+ // READY state transitions
+ {
+ SCOPED_TRACE("READY state transitions");
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_READY_ST));
+ }
+
+ // WAITING state transitions
+ {
+ SCOPED_TRACE("WAITING state transitions");
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_WAITING_ST));
+ }
+}
+
+// This test checks that the server in the hot standby mode does not
+// transition to the "syncing" state when "sync-leases" is disabled.
+// This is the primary server case.
+TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyPrimary) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->getPeerConfig("server2")->setRole("standby");
+ valid_config->setSyncLeases(false);
+
+ startService(valid_config);
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+}
+
+// This test checks that the primary server in the hot standby mode
+// transitions to the "terminated" state when the clock skew gets high.
+TEST_F(HAServiceStateMachineTest, terminateTransitionsHotStandbyPrimary) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ testTerminateTransition(MyState(HA_HOT_STANDBY_ST));
+ testTerminateTransition(MyState(HA_PARTNER_DOWN_ST));
+ testTerminateTransition(MyState(HA_READY_ST));
+ testTerminateTransition(MyState(HA_WAITING_ST));
+}
+
+// This test checks all combinations of server and partner states and the
+// resulting state to which the server transitions. This server is standby.
+// There is another test which validates state transitions from the
+// primary server perspective.
+TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ // HOT STANDBY state transitions
+ {
+ SCOPED_TRACE("HOT STANDBY state transitions");
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+ }
+
+ // in-maintenance state transitions
+ {
+ SCOPED_TRACE("in-maintenance state transitions");
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+ }
+
+ // PARTNER DOWN state transitions
+ {
+ SCOPED_TRACE("PARTNER DOWN state transitions");
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+
+ testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_PARTNER_DOWN_ST));
+ }
+
+
+ // READY state transitions
+ {
+ SCOPED_TRACE("READY state transitions");
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_HOT_STANDBY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_IN_MAINTENANCE_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_TERMINATED_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_READY_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_READY_ST));
+ }
+
+ // WAITING state transitions
+ {
+ SCOPED_TRACE("WAITING state transitions");
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_IN_MAINTENANCE_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_SYNCING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_SYNCING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_TERMINATED_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_WAITING_ST),
+ FinalState(HA_WAITING_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_UNAVAILABLE_ST),
+ FinalState(HA_WAITING_ST));
+ }
+}
+
+// This test checks that the server in the hot standby mode does not
+// transition to the "syncing" state when "sync-leases" is disabled.
+// This is the standby server case.
+TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyStandby) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+ valid_config->setSyncLeases(false);
+
+ startService(valid_config);
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
+ FinalState(HA_READY_ST));
+
+ testTransition(MyState(HA_WAITING_ST), PartnerState(HA_READY_ST),
+ FinalState(HA_READY_ST));
+}
+
+// This test checks that the standby server in the hot standby mode
+// transitions to the "terminated" state when the clock skew gets high.
+TEST_F(HAServiceStateMachineTest, terminateTransitionsHotStandbyStandby) {
+ partner_.reset(new HAPartner(listener_, factory_));
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+
+ // Turn it into hot-standby configuration.
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ testTerminateTransition(MyState(HA_HOT_STANDBY_ST));
+ testTerminateTransition(MyState(HA_PARTNER_DOWN_ST));
+ testTerminateTransition(MyState(HA_READY_ST));
+ testTerminateTransition(MyState(HA_WAITING_ST));
+}
+
+// This test verifies that the server takes ownership of the given scopes
+// and whether the DHCP service is disabled or enabled in certain states.
+// This is primary server.
+TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyPrimary) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ // HOT STANDBY and TERMINATED: serving my own scope
+ expectScopes(MyState(HA_HOT_STANDBY_ST), { "server1" }, true);
+ expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true);
+
+ // PARTNER DOWN and PARTNER IN MAINTENANCE: still serving my own scope.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true);
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies that auto-failover setting does not affect scopes
+// handling by the primary server in the hot-standby mode.
+TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyPrimaryNoFailover) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ // Disable auto-failover.
+ valid_config->getThisServerConfig()->setAutoFailover(false);
+
+ startService(valid_config);
+
+ // HOT STANDBY and TERMINATED: serving my own scope
+ expectScopes(MyState(HA_HOT_STANDBY_ST), { "server1" }, true);
+ expectScopes(MyState(HA_TERMINATED_ST), { "server1" }, true);
+
+ // PARTNER IN MAINTENANCE & PARTNER DOWN: still serving my own scope.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true);
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies if the server would send lease updates to the partner
+// while being in various states. The HA configuration is hot standby and
+// the server is primary.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesHotStandbyPrimary) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_HOT_STANDBY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// This test verifies if the server would send heartbeat to the partner
+// while being in various states. The HA configuration is hot standby.
+TEST_F(HAServiceStateMachineTest, heartbeatHotStandby) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_HOT_STANDBY_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_DOWN_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_PARTNER_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_READY_ST)));
+ EXPECT_FALSE(expectHeartbeat(MyState(HA_TERMINATED_ST)));
+ EXPECT_TRUE(expectHeartbeat(MyState(HA_WAITING_ST)));
+}
+
+// This test verifies that the server takes ownership of the given scopes
+// and whether the DHCP service is disabled or enabled in certain states.
+// This is standby server.
+TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyStandby) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ // HOT STANDBY: serving no scopes because the primary is active.
+ // The DHCP is enabled to be able to monitor activity of the
+ // primary.
+ expectScopes(MyState(HA_HOT_STANDBY_ST), { }, true);
+
+ // TERMINATED: serving no scopes because the primary is active.
+ expectScopes(MyState(HA_TERMINATED_ST), { }, true);
+
+ // PARTNER IN MAINTENANCE & PARTNER DOWN: serving server1's scope.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true);
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies that the standby server does not take ownership
+// of the primary server's scope when auto-failover is set to false
+TEST_F(HAServiceStateMachineTest, scopesServingHotStandbyStandbyNoFailover) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ // Disable auto-failover.
+ valid_config->getThisServerConfig()->setAutoFailover(false);
+
+ startService(valid_config);
+
+ // HOT STANDBY: serving no scopes because the primary is active.
+ // The DHCP is enabled to be able to monitor activity of the
+ // primary.
+ expectScopes(MyState(HA_HOT_STANDBY_ST), { }, true);
+
+ // TERMINATED: serving no scopes because the primary is active.
+ expectScopes(MyState(HA_TERMINATED_ST), { }, true);
+
+ // PARTNER DOWN: still serving no scopes because auto-failover is set to false.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { }, true);
+
+ // PARTNER IN MAINTENANCE: serving partner's scopes.
+ expectScopes(MyState(HA_PARTNER_IN_MAINTENANCE_ST), { "server1" }, true);
+
+ // Same for the partner-down case during maintenance.
+ expectScopes(MyState(HA_PARTNER_DOWN_ST), { "server1" }, true,
+ HAService::HA_MAINTENANCE_START_EVT);
+
+ // IN MAINTENANCE, READY & WAITING: serving no scopes.
+ expectScopes(MyState(HA_IN_MAINTENANCE_ST), { }, false);
+ expectScopes(MyState(HA_READY_ST), { }, false);
+ expectScopes(MyState(HA_WAITING_ST), { }, false);
+}
+
+// This test verifies if the server would send lease updates to the partner
+// while being in various states. The HA configuration is hot standby and
+// the server is secondary.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesHotStandbyStandby) {
+ HAConfigPtr valid_config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ valid_config->setThisServerName("server2");
+ valid_config->getPeerConfig("server2")->setRole("standby");
+
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getFailoverPeerConfig();
+
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_HOT_STANDBY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_PARTNER_DOWN_ST), peer_config));
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_PARTNER_IN_MAINTENANCE_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_READY_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_SYNCING_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_TERMINATED_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// This test verifies if the backup server doesn't send lease updates.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesBackup) {
+ HAConfigPtr valid_config = createValidConfiguration();
+
+ // Turn it into hot-standby configuration.
+ valid_config->setThisServerName("server3");
+
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getPeerConfig("server1");
+
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_BACKUP_ST), peer_config));
+ EXPECT_FALSE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// This test checks all combinations of the primary server in the passive
+// backup configuration.
+TEST_F(HAServiceStateMachineTest, stateTransitionsPassiveBackup) {
+ partner_->startup();
+
+ HAConfigPtr valid_config = createValidPassiveBackupConfiguration();
+ startService(valid_config);
+
+ testPassiveBackupTransition(MyState(HA_WAITING_ST), FinalState(HA_PASSIVE_BACKUP_ST));
+ testPassiveBackupTransition(MyState(HA_PASSIVE_BACKUP_ST), FinalState(HA_PASSIVE_BACKUP_ST));
+}
+
+// This test checks that no scopes are being served in the waiting state
+// of the passive backup configuration and that active server's scope is
+// served while in the passive-backup state.
+TEST_F(HAServiceStateMachineTest, scopesServingPassiveBackup) {
+ HAConfigPtr valid_config = createValidPassiveBackupConfiguration();
+ startService(valid_config);
+
+ EXPECT_TRUE(service_->query_filter_.amServingScope("server1"));
+ EXPECT_FALSE(service_->query_filter_.amServingScope("server2"));
+ EXPECT_TRUE(service_->network_state_->isServiceEnabled());
+}
+
+// Verifies that lease updates are sent by the server in the passive-backup state.
+TEST_F(HAServiceStateMachineTest, shouldSendLeaseUpdatesPassiveBackup) {
+ HAConfigPtr valid_config = createValidPassiveBackupConfiguration();
+ startService(valid_config);
+
+ HAConfig::PeerConfigPtr peer_config = valid_config->getPeerConfig("server2");
+
+ EXPECT_TRUE(expectLeaseUpdates(MyState(HA_WAITING_ST), peer_config));
+}
+
+// Verifies that the server transitions to the terminated state as a result
+// of too many rejected leases.
+TEST_F(HAServiceStateMachineTest, shouldTerminateDueToRejectedLeases) {
+ startService(createValidConfiguration());
+ service_->verboseTransition(HA_LOAD_BALANCING_ST);
+ service_->runModel(HAService::NOP_EVT);
+
+ // Simulate many rejected leases. It should cause the server to transition
+ // to the terminated state.
+ simulateRejectedLeaseUpdates(100);
+ EXPECT_EQ(HA_TERMINATED_ST, service_->getCurrState());
+}
+
+// Verifies that rejected leases are cleared upon entering certain states and
+// are not cleared upon entering other states in the load balancing mode.
+TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesLoadBalancing) {
+ startService(createValidConfiguration());
+
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_COMMUNICATION_RECOVERY_ST)));
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_LOAD_BALANCING_ST)));
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_PARTNER_DOWN_ST)));
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PARTNER_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_READY_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_TERMINATED_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_WAITING_ST)));
+}
+
+// Verifies that rejected leases are cleared upon entering certain states and
+// are not cleared upon entering other states in the hot standby mode.
+TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesHotStandby) {
+ HAConfigPtr config_storage = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config_storage->getPeerConfig("server2")->setRole("standby");
+ startService(config_storage);
+
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_HOT_STANDBY_ST)));
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_PARTNER_DOWN_ST)));
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PARTNER_IN_MAINTENANCE_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_READY_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_TERMINATED_ST)));
+ EXPECT_TRUE(expectClearRejectedLeases(MyState(HA_WAITING_ST)));
+}
+
+// Verifies that rejected leases are cleared upon entering certain states and
+// are not cleared upon entering other states in the passive backup mode.
+TEST_F(HAServiceStateMachineTest, clearRejectedLeaseUpdatesPassiveBackup) {
+ HAConfigPtr config_storage = createValidPassiveBackupConfiguration();
+ config_storage->getPeerConfig("server2")->setRole("backup");
+ startService(config_storage);
+
+ EXPECT_FALSE(expectClearRejectedLeases(MyState(HA_PASSIVE_BACKUP_ST)));
+}
+
+// Verifies that the server doesn't terminate when the partner is unavailable.
+TEST_F(HAServiceStateMachineTest, doNotTerminateWhenPartnerUnavailable) {
+ HAConfigPtr config_storage = createValidConfiguration();
+ ASSERT_NO_THROW_LOG(service_.reset(new TestHAService(io_service_, network_state_,
+ config_storage)));
+ NakedCommunicationState4Ptr state(new NakedCommunicationState4(io_service_,
+ config_storage));
+ service_->communication_state_ = state;
+
+ // Begin in the load-balancing state and the partner is also in
+ // that state.
+ service_->verboseTransition(HA_LOAD_BALANCING_ST);
+ service_->runModel(HAService::NOP_EVT);
+ service_->communication_state_->poke();
+ service_->communication_state_->setPartnerState("load-balancing");
+
+ // Change the clock skew, so it is beyond the acceptable threshold.
+ state->clock_skew_ += boost::posix_time::time_duration(0, 1, 5);
+
+ // Send the heartbeat. We didn't setup the connection to the partner.
+ // The heartbeat should fail and our server should transition to the
+ // communication-recovery state. Earlier, the server had a bug that
+ // it would take the last known clock skew and transition to the
+ // terminated state instead. It was wrong because when the partner is
+ // unavailable we can't tell the current clock skew. The administrator
+ // may actually be fixing the time on the other server. Thus, we must
+ // not terminate.
+ ASSERT_NO_FATAL_FAILURE(waitForEvent(HAService::HA_HEARTBEAT_COMPLETE_EVT));
+ EXPECT_EQ(HA_COMMUNICATION_RECOVERY_ST, service_->getCurrState());
+}
+
+} // end of anonymous namespace
diff --git a/src/hooks/dhcp/high_availability/tests/ha_test.cc b/src/hooks/dhcp/high_availability/tests/ha_test.cc
new file mode 100644
index 0000000..7c90242
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_test.cc
@@ -0,0 +1,378 @@
+// Copyright (C) 2017-2023 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <asiolink/asio_wrapper.h>
+#include <ha_test.h>
+#include <asiolink/interval_timer.h>
+#include <cc/command_interpreter.h>
+#include <cc/data.h>
+#include <dhcp/dhcp4.h>
+#include <dhcp/dhcp6.h>
+#include <dhcp/duid.h>
+#include <dhcp/hwaddr.h>
+#include <dhcp/option.h>
+#include <dhcp/option_int.h>
+#include <dhcpsrv/cfgmgr.h>
+#include <hooks/hooks.h>
+#include <hooks/hooks_manager.h>
+#include <util/range_utilities.h>
+#include <functional>
+#include <utility>
+#include <vector>
+
+using namespace isc::asiolink;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::hooks;
+
+namespace {
+
+/// @brief Structure that holds registered hook indexes.
+struct TestHooks {
+ int hooks_index_dhcp4_srv_configured_;
+
+ /// Constructor that registers hook points for the tests.
+ TestHooks() {
+ hooks_index_dhcp4_srv_configured_ = HooksManager::registerHook("dhcp4_srv_configured");
+ }
+};
+
+TestHooks Hooks;
+
+}
+
+namespace isc {
+namespace ha {
+namespace test {
+
+HATest::HATest()
+ : io_service_(new IOService()),
+ network_state_(new NetworkState(NetworkState::DHCPv4)) {
+}
+
+HATest::~HATest() {
+}
+
+void
+HATest::startHAService() {
+ if (HooksManager::calloutsPresent(Hooks.hooks_index_dhcp4_srv_configured_)) {
+ CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+ callout_handle->setArgument("io_context", io_service_);
+ callout_handle->setArgument("network_state", network_state_);
+ HooksManager::callCallouts(Hooks.hooks_index_dhcp4_srv_configured_,
+ *callout_handle);
+ }
+}
+
+void
+HATest::runIOService(long ms) {
+ io_service_->get_io_service().reset();
+ IntervalTimer timer(*io_service_);
+ timer.setup(std::bind(&IOService::stop, io_service_), ms,
+ IntervalTimer::ONE_SHOT);
+ io_service_->run();
+ timer.cancel();
+}
+
+void
+HATest::runIOService(long ms, std::function<bool()> stop_condition) {
+ io_service_->get_io_service().reset();
+ IntervalTimer timer(*io_service_);
+ bool timeout = false;
+ timer.setup(std::bind(&HATest::stopIOServiceHandler, this, std::ref(timeout)),
+ ms, IntervalTimer::ONE_SHOT);
+
+ while (!stop_condition() && !timeout) {
+ io_service_->run_one();
+ }
+
+ timer.cancel();
+}
+
+boost::shared_ptr<std::thread>
+HATest::runIOServiceInThread() {
+ io_service_->get_io_service().reset();
+
+ bool running = false;
+ std::mutex mutex;
+ std::condition_variable condvar;
+
+ io_service_->post(std::bind(&HATest::signalServiceRunning, this, std::ref(running),
+ std::ref(mutex), std::ref(condvar)));
+ boost::shared_ptr<std::thread>
+ th(new std::thread(std::bind(&IOService::run, io_service_.get())));
+
+ std::unique_lock<std::mutex> lock(mutex);
+ while (!running) {
+ condvar.wait(lock);
+ }
+
+ return (th);
+}
+
+void
+HATest::testSynchronousCommands(std::function<void()> commands) {
+ // Run IO service in thread.
+ auto thread = runIOServiceInThread();
+
+ // Run desired commands.
+ commands();
+
+ // Stop the IO service. This should cause the thread to terminate.
+ io_service_->stop();
+ thread->join();
+}
+
+void
+HATest::signalServiceRunning(bool& running, std::mutex& mutex,
+ std::condition_variable& condvar) {
+ std::lock_guard<std::mutex> lock(mutex);
+ running = true;
+ condvar.notify_one();
+}
+
+void
+HATest::stopIOServiceHandler(bool& stop_flag) {
+ stop_flag = true;
+}
+
+ConstElementPtr
+HATest::createValidJsonConfiguration(const HAConfig::HAMode& ha_mode) const {
+ std::ostringstream config_text;
+ config_text <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": " << (ha_mode == HAConfig::LOAD_BALANCING ?
+ "\"load-balancing\"" : "\"hot-standby\"") << ","
+ " \"sync-page-limit\": 3,"
+ " \"heartbeat-delay\": 1000,"
+ " \"max-response-delay\": 1000,"
+ " \"max-ack-delay\": 10000,"
+ " \"max-unacked-clients\": 10,"
+ " \"max-rejected-clients\": 10,"
+ " \"multi-threading\": {"
+ " \"enable-multi-threading\": false"
+ " },"
+ " \"wait-backup-ack\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:18123/\","
+ " \"role\": \"primary\","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:18124/\","
+ " \"role\": " << (ha_mode == HAConfig::LOAD_BALANCING ?
+ "\"secondary\"" : "\"standby\"") << ","
+ " \"auto-failover\": true"
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:18125/\","
+ " \"role\": \"backup\","
+ " \"auto-failover\": false"
+ " }"
+ " ]"
+ " }"
+ "]";
+
+ return (Element::fromJSON(config_text.str()));
+}
+
+ConstElementPtr
+HATest::createValidPassiveBackupJsonConfiguration() const {
+ std::ostringstream config_text;
+ config_text <<
+ "["
+ " {"
+ " \"this-server-name\": \"server1\","
+ " \"mode\": \"passive-backup\","
+ " \"multi-threading\": {"
+ " \"enable-multi-threading\": false"
+ " },"
+ " \"wait-backup-ack\": false,"
+ " \"peers\": ["
+ " {"
+ " \"name\": \"server1\","
+ " \"url\": \"http://127.0.0.1:18123/\","
+ " \"role\": \"primary\""
+ " },"
+ " {"
+ " \"name\": \"server2\","
+ " \"url\": \"http://127.0.0.1:18124/\","
+ " \"role\": \"backup\""
+ " },"
+ " {"
+ " \"name\": \"server3\","
+ " \"url\": \"http://127.0.0.1:18125/\","
+ " \"role\": \"backup\""
+ " }"
+ " ]"
+ " }"
+ "]";
+
+ return (Element::fromJSON(config_text.str()));
+}
+
+
+HAConfigPtr
+HATest::createValidConfiguration(const HAConfig::HAMode& ha_mode) const {
+ HAConfigPtr config_storage(new HAConfig());
+ HAConfigParser parser;
+
+ parser.parse(config_storage, createValidJsonConfiguration(ha_mode));
+ return (config_storage);
+}
+
+HAConfigPtr
+HATest::createValidPassiveBackupConfiguration() const {
+ HAConfigPtr config_storage(new HAConfig());
+ HAConfigParser parser;
+
+ parser.parse(config_storage, createValidPassiveBackupJsonConfiguration());
+ return (config_storage);
+}
+
+void
+HATest::checkAnswer(const isc::data::ConstElementPtr& answer,
+ const int exp_status,
+ const std::string& exp_txt) {
+ int rcode = 0;
+ isc::data::ConstElementPtr comment;
+ comment = isc::config::parseAnswer(rcode, answer);
+
+ EXPECT_EQ(exp_status, rcode)
+ << "Expected status code " << exp_status
+ << " but received " << rcode << ", comment: "
+ << (comment ? comment->str() : "(none)");
+
+ // Ok, parseAnswer interface is weird. If there are no arguments,
+ // it returns content of text. But if there is an argument,
+ // it returns the argument and it's not possible to retrieve
+ // "text" (i.e. comment).
+ if (comment->getType() != Element::string) {
+ comment = answer->get("text");
+ }
+
+ if (!exp_txt.empty()) {
+ EXPECT_EQ(exp_txt, comment->stringValue());
+ }
+}
+
+std::vector<uint8_t>
+HATest::randomKey(const size_t key_size) const {
+ std::vector<uint8_t> key(key_size);
+ util::fillRandom(key.begin(), key.end());
+ return (key);
+}
+
+
+Pkt4Ptr
+HATest::createMessage4(const uint8_t msg_type, const uint8_t hw_address_seed,
+ const uint8_t client_id_seed, const uint16_t secs) const {
+ Pkt4Ptr message(new Pkt4(msg_type, 0x1234));
+
+ HWAddrPtr hw_address(new HWAddr(std::vector<uint8_t>(6, hw_address_seed),
+ HTYPE_ETHER));
+ message->setHWAddr(hw_address);
+ message->setSecs(secs);
+
+ if (client_id_seed > 0) {
+ OptionPtr opt_client_id(new Option(Option::V4, DHO_DHCP_CLIENT_IDENTIFIER,
+ std::vector<uint8_t>(6, client_id_seed)));
+ message->addOption(opt_client_id);
+ }
+
+ return (message);
+}
+
+Pkt4Ptr
+HATest::createQuery4(const std::string& hw_address_text) const {
+ Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 0x1234));
+ HWAddr hwaddr = HWAddr::fromText(hw_address_text);
+ query4->setHWAddr(HWAddrPtr(new HWAddr(hwaddr.hwaddr_, HTYPE_ETHER)));
+ return (query4);
+}
+
+Pkt4Ptr
+HATest::createQuery4(const std::vector<uint8_t>& hw_address,
+ const std::vector<uint8_t>& client_id) const {
+ Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 0x1234));
+ query4->setHWAddr(HWAddrPtr(new HWAddr(hw_address, HTYPE_ETHER)));
+ if (!client_id.empty()) {
+ OptionPtr opt_client_id(new Option(Option::V4, DHO_DHCP_CLIENT_IDENTIFIER,
+ client_id));
+ query4->addOption(opt_client_id);
+ }
+ return (query4);
+}
+
+Pkt6Ptr
+HATest::createQuery6(const std::vector<uint8_t>& duid) const {
+ Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 0x1234));
+ OptionPtr opt_duid(new Option(Option::V6, D6O_CLIENTID, duid));
+ query6->addOption(opt_duid);
+ return (query6);
+}
+
+Pkt6Ptr
+HATest::createMessage6(const uint8_t msg_type, const uint8_t duid_seed,
+ const uint16_t elapsed_time) const {
+ Pkt6Ptr message(new Pkt6(msg_type, 0x1234));
+
+ OptionPtr opt_duid(new Option(Option::V6, D6O_CLIENTID, OptionBuffer(8, duid_seed)));
+ message->addOption(opt_duid);
+
+ OptionUint16Ptr opt_elapsed_time(new OptionUint16(Option::V6, D6O_ELAPSED_TIME,
+ elapsed_time));
+ message->addOption(opt_elapsed_time);
+
+ return (message);
+}
+
+Pkt6Ptr
+HATest::createQuery6(const std::string& duid_text) const {
+ Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 0x1234));
+ DUID duid = DUID::fromText(duid_text);
+ OptionPtr client_id(new Option(Option::V6, D6O_CLIENTID, duid.getDuid()));
+ query6->addOption(client_id);
+ return (query6);
+}
+
+void
+HATest::setDHCPMultiThreadingConfig(bool enabled, uint32_t thread_pool_size,
+ uint32_t packet_queue_size) {
+ ElementPtr mt_config = Element::createMap();
+ mt_config->set("enable-multi-threading", Element::create(enabled));
+ mt_config->set("thread-pool-size",
+ Element::create(static_cast<int>(thread_pool_size)));
+ mt_config->set("queue-size",
+ Element::create(static_cast<int>(packet_queue_size)));
+ CfgMgr::instance().getStagingCfg()->setDHCPMultiThreading(mt_config);
+}
+
+std::string
+HATest::makeHAMtJson(bool enable_multi_threading, bool http_dedicated_listener,
+ uint32_t http_listener_threads, uint32_t http_client_threads) {
+ std::stringstream ss;
+ ss << "\"multi-threading\": {"
+ << " \"enable-multi-threading\": "
+ << (enable_multi_threading ? "true" : "false") << ","
+ << " \"http-dedicated-listener\": "
+ << (http_dedicated_listener ? "true" : "false") << ","
+ << " \"http-listener-threads\": " << http_listener_threads << ","
+ << " \"http-client-threads\": " << http_client_threads << "}";
+ return (ss.str());
+}
+
+} // namespace test
+} // namespace ha
+} // namespace isc
diff --git a/src/hooks/dhcp/high_availability/tests/ha_test.h b/src/hooks/dhcp/high_availability/tests/ha_test.h
new file mode 100644
index 0000000..0b763ca
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/ha_test.h
@@ -0,0 +1,304 @@
+// Copyright (C) 2017-2022 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <communication_state.h>
+#include <ha_config.h>
+#include <ha_config_parser.h>
+#include <asiolink/io_service.h>
+#include <cc/data.h>
+#include <dhcp/duid.h>
+#include <dhcp/hwaddr.h>
+#include <dhcp/pkt4.h>
+#include <dhcp/pkt6.h>
+#include <dhcpsrv/network_state.h>
+#include <hooks/libinfo.h>
+#include <boost/shared_ptr.hpp>
+#include <gtest/gtest.h>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+
+namespace isc {
+namespace ha {
+namespace test {
+
+/// @brief Derivation of the @c CommunicationState class which allows
+/// for modifications of poke time.
+///
+/// @tparam @c CommunicationState4 or @c CommunicationState6.
+template<typename StateType>
+class NakedCommunicationState : public StateType {
+public:
+
+ /// @brief Constructor.
+ ///
+ /// @param io_service pointer to the IO service object.
+ explicit NakedCommunicationState(const asiolink::IOServicePtr& io_service,
+ const HAConfigPtr& config)
+ : StateType(io_service, config) {
+ }
+
+ /// @brief Checks if the object was poked recently.
+ ///
+ /// @return true if the object was poked less than 5 seconds ago,
+ /// false otherwise.
+ bool isPoked() const {
+ return (StateType::getDurationInMillisecs() < 5000);
+ }
+
+ using StateType::config_;
+ using StateType::timer_;
+ using StateType::clock_skew_;
+ using StateType::last_clock_skew_warn_;
+ using StateType::my_time_at_skew_;
+ using StateType::partner_time_at_skew_;
+ using StateType::unsent_update_count_;
+ using StateType::getRejectedLeaseUpdatesCountFromContainer;
+};
+
+/// @brief Type of the NakedCommunicationState for DHCPv4.
+typedef NakedCommunicationState<CommunicationState4> NakedCommunicationState4;
+
+/// @brief Type of the pointer to the @c NakedCommunicationState4.
+typedef boost::shared_ptr<NakedCommunicationState4> NakedCommunicationState4Ptr;
+
+/// @brief Type of the NakedCommunicationState for DHCPv6.
+typedef NakedCommunicationState<CommunicationState6> NakedCommunicationState6;
+
+/// @brief Type of the pointer to the @c NakedCommunicationState6.
+typedef boost::shared_ptr<NakedCommunicationState6> NakedCommunicationState6Ptr;
+
+/// @brief General test fixture class for all HA unittests.
+///
+/// It provides basic functions to load and unload HA hooks library.
+/// All test classes should derive from this class.
+class HATest : public ::testing::Test {
+public:
+
+ /// @brief Constructor.
+ HATest();
+
+ /// @brief Destructor.
+ virtual ~HATest();
+
+ /// @brief Calls dhcp4_srv_configured callout to set IO service pointer.
+ void startHAService();
+
+ /// @brief Runs IO service for a specified amount of time.
+ ///
+ /// @param ms number of milliseconds for which the IO service should be
+ /// run.
+ void runIOService(long ms);
+
+ /// @brief Runs IO service until timeout occurs or until provided method
+ /// returns true.
+ ///
+ /// @param ms number of milliseconds for which the IO service should be
+ /// run.
+ /// @param stop_condition pointer to the function which returns true if
+ /// when the IO service should be stopped.
+ void runIOService(long ms, std::function<bool()> stop_condition);
+
+ /// @brief Runs IO service in a thread.
+ ///
+ /// @return Shared pointer to the thread.
+ boost::shared_ptr<std::thread> runIOServiceInThread();
+
+ /// @brief Executes commands while running IO service in a thread.
+ ///
+ /// @param commands pointer to a function to be executed while IO service
+ /// is run in thread.
+ void testSynchronousCommands(std::function<void()> commands);
+
+protected:
+
+ /// @brief Signals that the IO service is running.
+ ///
+ /// @param running reference to the flag which is set to true when the
+ /// IO service starts running and executes this function.
+ /// @param mutex reference to the mutex used for synchronization.
+ /// @param condvar reference to condition variable used for synchronization.
+ void signalServiceRunning(bool& running, std::mutex& mutex,
+ std::condition_variable& condvar);
+
+public:
+
+ /// @brief Handler for timeout during runIOService invocation.
+ ///
+ /// @param [out] stop_flag set to true when the handler is invoked.
+ void stopIOServiceHandler(bool& stop_flag);
+
+ /// @brief Return HA configuration with three servers in JSON format.
+ ///
+ /// @param ha_mode HA operation mode (default is load balancing).
+ /// @return Pointer to the unparsed configuration.
+ data::ConstElementPtr
+ createValidJsonConfiguration(const HAConfig::HAMode& ha_mode =
+ HAConfig::LOAD_BALANCING) const;
+
+ /// @brief Return HA configuration with one primary and two backup
+ /// servers in the JSON format.
+ ///
+ /// @return Pointer to the unparsed configuration.
+ data::ConstElementPtr
+ createValidPassiveBackupJsonConfiguration() const;
+
+ /// @brief Return HA configuration with three servers.
+ ///
+ /// @param ha_mode HA operation mode (default is load balancing).
+ /// @return Pointer to the parsed configuration.
+ HAConfigPtr createValidConfiguration(const HAConfig::HAMode& ha_mode =
+ HAConfig::LOAD_BALANCING) const;
+
+ /// @brief Return passive-backup configuration.
+ ///
+ /// @return Pointer to the parsed configuration.
+ HAConfigPtr createValidPassiveBackupConfiguration() const;
+
+ /// @brief Checks the status code and message against expected values.
+ ///
+ /// @param answer Element set containing an integer response and string
+ /// comment.
+ /// @param exp_status Status code against which to compare the status.
+ /// @param exp_txt Expected text (not checked if empty)
+ void checkAnswer(const isc::data::ConstElementPtr& answer,
+ const int exp_status,
+ const std::string& exp_txt = "");
+
+ /// @brief Creates an identifier of arbitrary size with random values.
+ ///
+ /// This function is useful in generating random client identifiers and
+ /// HW addresses for load balancing tests.
+ ///
+ /// @param key_size Size of the generated random identifier.
+ std::vector<uint8_t> randomKey(const size_t key_size) const;
+
+ /// @brief Generates simple DHCPv4 message.
+ ///
+ /// @param msg_type DHCPv4 message type to be created.
+ /// @param hw_address_seed value from which HW address will be generated.
+ /// @param client_id_seed value from which client identifier will be
+ /// generated.
+ /// @param secs value to be stored in the "secs" field of the DHCPv4 message.
+ ///
+ /// @return Pointer to the created message.
+ dhcp::Pkt4Ptr createMessage4(const uint8_t msg_type,
+ const uint8_t hw_address_seed,
+ const uint8_t client_id_seed,
+ const uint16_t secs) const;
+
+ /// @brief Creates test DHCPv4 query instance.
+ ///
+ /// @param hw_address_text HW address to be included in the query. It is
+ /// used in load balancing.
+ ///
+ /// @return Pointer to the DHCPv4 query instance.
+ dhcp::Pkt4Ptr createQuery4(const std::string& hw_address_text) const;
+
+ /// @brief Creates test DHCPv4 query instance.
+ ///
+ /// @param hw_address HW address to be included in the query. It is used
+ /// in load balancing.
+ /// @param client_id optional client identifier.
+ dhcp::Pkt4Ptr createQuery4(const std::vector<uint8_t>& hw_address,
+ const std::vector<uint8_t>& client_id =
+ std::vector<uint8_t>()) const;
+
+ /// @brief Creates test DHCPv6 query instance.
+ ///
+ /// @param duid DUI to be included in the query. It is used in load balancing.
+ dhcp::Pkt6Ptr createQuery6(const std::vector<uint8_t>& duid) const;
+
+ /// @brief Generates simple DHCPv6 message.
+ ///
+ /// @param msg_type DHCPv6 message type to be created.
+ /// @param duid value from which DUID will be generated.
+ /// @param elapsed_time value of the Elapsed Time option.
+ ///
+ /// @return Pointer to the created message.
+ dhcp::Pkt6Ptr createMessage6(const uint8_t msg_type,
+ const uint8_t duid_seed,
+ const uint16_t elapsed_time) const;
+
+ /// @brief Sets the DHCP multi-threading configuration in staging SrvConfig.
+ ///
+ /// @param enable_multi_threading value that maps to enable-multi-threading.
+ /// @param thread_pool_size value that maps to thread-pool-size.
+ /// @param queue_size value that maps to queue-size.
+ void setDHCPMultiThreadingConfig(bool enable_multi_threading,
+ uint32_t thread_pool_size = 0,
+ uint32_t queue_size = 16);
+
+ /// @brief Replace a pattern in a configuration.
+ ///
+ /// @param config Configuration to patch.
+ /// @param from String to replace.
+ /// @param repl String which replaces all occurrences of from.
+ /// @result A copy of config where all occurrences of from were replaced
+ /// by repl.
+ std::string replaceInConfig(const std::string& config,
+ const std::string& from,
+ const std::string& repl) {
+ std::string result(config);
+ if (from.empty()) {
+ return (result);
+ }
+ for (;;) {
+ size_t where = result.find(from);
+ if (where == std::string::npos) {
+ return (result);
+ }
+ result.replace(where, from.size(), repl);
+ }
+ return (result);
+ }
+
+ /// @brief Constructs JSON string for HA "multi-threading" element.
+ ///
+ /// Constructs a JSON string with the following content:
+ ///
+ /// ```
+ /// "multi-threading" {
+ /// "enable-multi-threading": <bool>,
+ /// "dedicated-http-listener": <bool>,
+ /// "http-listener-threads": <int>,
+ /// "http-client-threads": <int>
+ /// }"
+ /// ```
+ ///
+ /// @param enable_multi_threading value for enable-multi-threading.
+ /// @param http_dedicated_listener value for dedicated-http-listener.
+ /// @param http_listener_threads value for http-listener-threads
+ /// @param http_client_threads value for http-client-threads
+ ///
+ /// @return JSON string
+ std::string makeHAMtJson(bool enable_multi_threading,
+ bool http_dedicated_listener,
+ uint32_t http_listener_threads,
+ uint32_t http_client_threads);
+
+ /// @brief Creates test DHCPv6 query instance.
+ ///
+ /// @param duid_text DUID to be included in the query. It is used in load
+ /// balancing.
+ ///
+ /// @return Pointer to the DHCPv6 query instance.
+ dhcp::Pkt6Ptr createQuery6(const std::string& duid_text) const;
+
+ /// @brief Pointer to the IO service used in the tests.
+ asiolink::IOServicePtr io_service_;
+
+ /// @brief Object holding a state of the DHCP service.
+ dhcp::NetworkStatePtr network_state_;
+};
+
+} // end of namespace isc::ha::test
+} // end of namespace isc::ha
+} // end of namespace isc
diff --git a/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc b/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc
new file mode 100644
index 0000000..a1cbe62
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/lease_update_backlog_unittest.cc
@@ -0,0 +1,91 @@
+// Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <lease_update_backlog.h>
+#include <asiolink/io_address.h>
+#include <dhcp/duid.h>
+#include <dhcp/hwaddr.h>
+
+#include <boost/make_shared.hpp>
+#include <boost/pointer_cast.hpp>
+#include <gtest/gtest.h>
+
+using namespace isc::asiolink;
+using namespace isc::dhcp;
+using namespace isc::ha;
+
+namespace {
+
+// This test verifies that lease updates can be pushed to the queue,
+// retrieved from the queue.
+TEST(LeaseUpdateBacklogTest, pushAndPop) {
+ // Create the queue with limit of 5 lease updates.
+ LeaseUpdateBacklog backlog(5);
+
+ // Add 5 lease updates.
+ for (auto i = 0; i < 5; ++i) {
+ IOAddress address(i + 1);
+ HWAddrPtr hwaddr = boost::make_shared<HWAddr>(std::vector<uint8_t>(6, static_cast<uint8_t>(i)),
+ HTYPE_ETHER);
+ Lease4Ptr lease = boost::make_shared<Lease4>(address, hwaddr, ClientIdPtr(), 60, 0, 1);
+ // Some lease updates have type "Add", some have type "Delete".
+ ASSERT_TRUE(backlog.push(i % 2 ? LeaseUpdateBacklog::ADD : LeaseUpdateBacklog::DELETE, lease));
+ EXPECT_FALSE(backlog.wasOverflown());
+ }
+
+ // Add 6th lease update. This should exceed the size limit.
+ IOAddress address("192.0.2.0");
+ HWAddrPtr hwaddr = boost::make_shared<HWAddr>(std::vector<uint8_t>(6, static_cast<uint8_t>(0xA)),
+ HTYPE_ETHER);
+ Lease4Ptr lease = boost::make_shared<Lease4>(address, hwaddr, ClientIdPtr(), 60, 0, 1);
+ ASSERT_FALSE(backlog.push(LeaseUpdateBacklog::ADD, lease));
+ EXPECT_TRUE(backlog.wasOverflown());
+
+ // Try to pop all lease updates.
+ LeaseUpdateBacklog::OpType op_type;
+ for (auto i = 0; i < 5; ++i) {
+ auto lease = backlog.pop(op_type);
+ ASSERT_TRUE(lease);
+ ASSERT_EQ(i % 2 ? LeaseUpdateBacklog::ADD : LeaseUpdateBacklog::DELETE, op_type);
+ }
+
+ // When trying to pop from an empty queue it should return null pointer.
+ lease = boost::dynamic_pointer_cast<Lease4>(backlog.pop(op_type));
+ EXPECT_FALSE(lease);
+ EXPECT_TRUE(backlog.wasOverflown());
+
+ // Reset queue state.
+ ASSERT_NO_THROW(backlog.clear());
+ EXPECT_FALSE(backlog.wasOverflown());
+}
+
+// This test verifies that all lease updates can be removed.
+TEST(LeaseUpdateBacklogTest, clear) {
+ // Create the queue with limit of 5 lease updates.
+ LeaseUpdateBacklog backlog(5);
+
+ // Add 5 lease updates.
+ for (auto i = 0; i < 3; ++i) {
+ IOAddress address(i + 1);
+ HWAddrPtr hwaddr = boost::make_shared<HWAddr>(std::vector<uint8_t>(6, static_cast<uint8_t>(i)),
+ HTYPE_ETHER);
+ Lease4Ptr lease = boost::make_shared<Lease4>(address, hwaddr, ClientIdPtr(), 60, 0, 1);
+ ASSERT_TRUE(backlog.push(LeaseUpdateBacklog::ADD, lease));
+ }
+
+ // Make sure all lease updates have been added.
+ EXPECT_EQ(3, backlog.size());
+
+ // Remove lease updates.
+ ASSERT_NO_THROW(backlog.clear());
+
+ // There should be no lease updates.
+ EXPECT_EQ(0, backlog.size());
+}
+
+} // end of anonymous namespace
diff --git a/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc b/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc
new file mode 100644
index 0000000..4f7f2c4
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/query_filter_unittest.cc
@@ -0,0 +1,1029 @@
+// Copyright (C) 2018-2022 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <ha_test.h>
+#include <ha_config.h>
+#include <ha_config_parser.h>
+#include <query_filter.h>
+#include <cc/data.h>
+#include <exceptions/exceptions.h>
+#include <dhcp/dhcp4.h>
+#include <dhcp/dhcp6.h>
+#include <dhcp/hwaddr.h>
+#include <util/multi_threading_mgr.h>
+
+#include <cstdint>
+#include <string>
+
+using namespace isc;
+using namespace isc::data;
+using namespace isc::dhcp;
+using namespace isc::ha;
+using namespace isc::ha::test;
+using namespace util;
+
+namespace {
+
+/// @brief Test fixture class for @c QueryFilter class.
+class QueryFilterTest : public HATest {
+public:
+ /// @brief Constructor.
+ QueryFilterTest() {
+ MultiThreadingMgr::instance().setMode(false);
+ }
+
+ /// @brief Destructor.
+ ~QueryFilterTest() {
+ MultiThreadingMgr::instance().setMode(false);
+ }
+
+ /// @brief This test verifies that client identifier is used for load
+ /// balancing.
+ void loadBalancingClientIdThisPrimary();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is primary.
+ void loadBalancingThisPrimary();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is secondary.
+ void loadBalancingThisSecondary();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is backup.
+ /// @todo Expand these tests once we implement the actual load balancing to
+ /// verify which packets are in scope.
+ void loadBalancingThisBackup();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is primary.
+ void hotStandbyThisPrimary();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is standby.
+ void hotStandbyThisSecondary();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is backup.
+ void hotStandbyThisBackup();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is primary.
+ void loadBalancingThisPrimary6();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is secondary.
+ void loadBalancingThisSecondary6();
+
+ /// @brief This test verifies the case when load balancing is enabled and
+ /// this server is backup.
+ /// @todo Expand these tests once we implement the actual load balancing to
+ /// verify which packets are in scope.
+ void loadBalancingThisBackup6();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is primary.
+ void hotStandbyThisPrimary6();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is standby.
+ void hotStandbyThisSecondary6();
+
+ /// @brief This test verifies the case when hot-standby is enabled and this
+ /// server is backup.
+ void hotStandbyThisBackup6();
+
+ /// @brief This test verifies that it is possible to explicitly enable and
+ /// disable certain scopes.
+ void explicitlyServeScopes();
+
+ /// @brief This test verifies that load balancing only affects the scope of
+ /// DHCPv4 message types that HA cares about.
+ void loadBalancingHaTypes4();
+
+ /// @brief This test verifies that load balancing only affects the scope of
+ /// DHCPv6 message types that HA cares about.
+ void loadBalancingHaTypes6();
+};
+
+void
+QueryFilterTest::loadBalancingThisPrimary() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // By default the server1 should serve its own scope only. The
+ // server2 should serve its scope.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // If the query is in scope, increase the counter of packets in scope.
+ if (filter.inScope(query4, scope_class)) {
+ ASSERT_EQ("HA_server1", scope_class);
+ ASSERT_NE(scope_class, "HA_server2");
+ ++in_scope;
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+
+ // Simulate failover scenario.
+ filter.serveFailoverScopes();
+
+ // In the failover case, the server1 should also take responsibility for
+ // the server2's queries.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // Every single query mist be in scope.
+ ASSERT_TRUE(filter.inScope(query4, scope_class));
+ }
+
+ // However, the one that lacks HW address and client id should be out of
+ // scope.
+ Pkt4Ptr query4(new Pkt4(DHCPDISCOVER, 1234));
+ EXPECT_FALSE(filter.inScope(query4, scope_class));
+}
+
+void
+QueryFilterTest::loadBalancingClientIdThisPrimary() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // By default the server1 should serve its own scope only. The
+ // server2 should serve its scope.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Fixed HW address used in tests.
+ std::vector<uint8_t> hw_address(HWAddr::ETHERNET_HWADDR_LEN);
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random client identifier.
+ Pkt4Ptr query4 = createQuery4(hw_address, randomKey(8));
+ // If the query is in scope, increase the counter of packets in scope.
+ if (filter.inScope(query4, scope_class)) {
+ ASSERT_EQ("HA_server1", scope_class);
+ ASSERT_NE(scope_class, "HA_server2");
+ ++in_scope;
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+
+ // Simulate failover scenario.
+ filter.serveFailoverScopes();
+
+ // In the failover case, the server1 should also take responsibility for
+ // the server2's queries.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random client identifier.
+ Pkt4Ptr query4 = createQuery4(hw_address, randomKey(8));
+ // Every single query mist be in scope.
+ ASSERT_TRUE(filter.inScope(query4, scope_class));
+ }
+}
+
+void
+QueryFilterTest::loadBalancingThisSecondary() {
+ HAConfigPtr config = createValidConfiguration();
+
+ // We're now a secondary server.
+ config->setThisServerName("server2");
+
+ QueryFilter filter(config);
+
+ // By default the server2 should serve its own scope only. The
+ // server1 should serve its scope.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // If the query is in scope, increase the counter of packets in scope.
+ if (filter.inScope(query4, scope_class)) {
+ ASSERT_EQ("HA_server2", scope_class);
+ ASSERT_NE(scope_class, "HA_server1");
+ ++in_scope;
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+
+ // Simulate failover scenario.
+ filter.serveFailoverScopes();
+
+ // In this scenario, the server1 died, so the server2 should now serve
+ // both scopes.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // Every single query must be in scope.
+ ASSERT_TRUE(filter.inScope(query4, scope_class));
+ }
+}
+
+void
+QueryFilterTest::loadBalancingThisBackup() {
+ HAConfigPtr config = createValidConfiguration();
+
+ config->setThisServerName("server3");
+
+ QueryFilter filter(config);
+
+ // The backup server doesn't handle any DHCP traffic by default.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // None of the packets should be handlded by the backup server.
+ ASSERT_FALSE(filter.inScope(query4, scope_class));
+ }
+
+ // Simulate failover. Although, backup server never starts handling
+ // other server's traffic automatically, it can be manually instructed
+ // to do so. This simulates such scenario.
+ filter.serveFailoverScopes();
+
+ // The backup server now handles traffic of server 1 and server 2.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // Every single query must be in scope.
+ ASSERT_TRUE(filter.inScope(query4, scope_class));
+ }
+}
+
+void
+QueryFilterTest::hotStandbyThisPrimary() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+
+ QueryFilter filter(config);
+
+ Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66");
+
+ // By default, only the primary server is active.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ // It should process its queries.
+ EXPECT_TRUE(filter.inScope(query4, scope_class));
+
+ // Simulate failover scenario, in which the active server detects a
+ // failure of the standby server. This doesn't change anything in how
+ // the traffic is distributed.
+ filter.serveFailoverScopes();
+
+ // The server1 continues to process its own traffic.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query4, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+}
+
+void
+QueryFilterTest::hotStandbyThisSecondary() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+ config->setThisServerName("server2");
+
+ QueryFilter filter(config);
+
+ Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66");
+
+ // The server2 doesn't process any queries by default. The whole
+ // traffic is processed by the server1.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ EXPECT_FALSE(filter.inScope(query4, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+
+ // Simulate failover case whereby the standby server detects a
+ // failure of the active server.
+ filter.serveFailoverScopes();
+
+ // The server2 now handles the traffic normally handled by the
+ // server1.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query4, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+}
+
+void
+QueryFilterTest::hotStandbyThisBackup() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+ config->setThisServerName("server3");
+
+ QueryFilter filter(config);
+
+ Pkt4Ptr query4 = createQuery4("11:22:33:44:55:66");
+
+ // By default the backup server doesn't process any traffic.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ EXPECT_FALSE(filter.inScope(query4, scope_class));
+
+ // Simulate failover. Although, backup server never starts handling
+ // other server's traffic automatically, it can be manually instructed
+ // to do so. This simulates such scenario.
+ filter.serveFailoverScopes();
+
+ // The backup server now handles the entire traffic, i.e. the traffic
+ // that the primary server handles.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query4, scope_class));
+}
+
+void
+QueryFilterTest::loadBalancingThisPrimary6() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // By default the server1 should serve its own scope only. The
+ // server2 should serve its scope.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random DUID.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ // If the query is in scope, increase the counter of packets in scope.
+ if (filter.inScope(query6, scope_class)) {
+ ASSERT_EQ("HA_server1", scope_class);
+ ASSERT_NE(scope_class, "HA_server2");
+ ++in_scope;
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+
+ // Simulate failover scenario.
+ filter.serveFailoverScopes();
+
+ // In the failover case, the server1 should also take responsibility for
+ // the server2's queries.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ // Every single query mist be in scope.
+ ASSERT_TRUE(filter.inScope(query6, scope_class));
+ }
+
+ // However, the one that lacks DUID should be out of scope.
+ Pkt6Ptr query6(new Pkt6(DHCPV6_SOLICIT, 1234));
+ EXPECT_FALSE(filter.inScope(query6, scope_class));
+}
+
+void
+QueryFilterTest::loadBalancingThisSecondary6() {
+ HAConfigPtr config = createValidConfiguration();
+
+ // We're now a secondary server.
+ config->setThisServerName("server2");
+
+ QueryFilter filter(config);
+
+ // By default the server2 should serve its own scope only. The
+ // server1 should serve its scope.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Count number of in scope packets.
+ unsigned in_scope = 0;
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ // If the query is in scope, increase the counter of packets in scope.
+ if (filter.inScope(query6, scope_class)) {
+ ASSERT_EQ("HA_server2", scope_class);
+ ASSERT_NE(scope_class, "HA_server1");
+ ++in_scope;
+ }
+ }
+
+ // We should have roughly 50/50 split of in scope and out of scope queries.
+ // However, we don't know exactly how many. To be safe we simply assume that
+ // we got more than 25% of in scope and more than 25% out of scope queries.
+ EXPECT_GT(in_scope, static_cast<unsigned>(queries_num / 4));
+ EXPECT_GT(queries_num - in_scope, static_cast<unsigned>(queries_num / 4));
+
+ // Simulate failover scenario.
+ filter.serveFailoverScopes();
+
+ // In this scenario, the server1 died, so the server2 should now serve
+ // both scopes.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt6Ptr query6 = createQuery6(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // Every single query must be in scope.
+ ASSERT_TRUE(filter.inScope(query6, scope_class));
+ }
+}
+
+void
+QueryFilterTest::loadBalancingThisBackup6() {
+ HAConfigPtr config = createValidConfiguration();
+
+ config->setThisServerName("server3");
+
+ QueryFilter filter(config);
+
+ // The backup server doesn't handle any DHCP traffic by default.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Set the test size - 65535 queries.
+ const unsigned queries_num = 65535;
+ std::string scope_class;
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ // None of the packets should be handlded by the backup server.
+ ASSERT_FALSE(filter.inScope(query6, scope_class));
+ }
+
+ // Simulate failover. Although, backup server never starts handling
+ // other server's traffic automatically, it can be manually instructed
+ // to do so. This simulates such scenario.
+ filter.serveFailoverScopes();
+
+ // The backup server now handles traffic of server 1 and server 2.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Repeat the test, but this time all should be in scope.
+ for (unsigned i = 0; i < queries_num; ++i) {
+ // Create query with random HW address.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ // Every single query must be in scope.
+ ASSERT_TRUE(filter.inScope(query6, scope_class));
+ }
+}
+
+void
+QueryFilterTest::hotStandbyThisPrimary6() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+
+ QueryFilter filter(config);
+
+ Pkt6Ptr query6 = createQuery6("01:02:11:22:33:44:55:66");
+
+ // By default, only the primary server is active.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ // It should process its queries.
+ EXPECT_TRUE(filter.inScope(query6, scope_class));
+
+ // Simulate failover scenario, in which the active server detects a
+ // failure of the standby server. This doesn't change anything in how
+ // the traffic is distributed.
+ filter.serveFailoverScopes();
+
+ // The server1 continues to process its own traffic.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query6, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+}
+
+void
+QueryFilterTest::hotStandbyThisSecondary6() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+ config->setThisServerName("server2");
+
+ QueryFilter filter(config);
+
+ Pkt6Ptr query6 = createQuery6("01:02:11:22:33:44:55:66");
+
+ // The server2 doesn't process any queries by default. The whole
+ // traffic is processed by the server1.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ EXPECT_FALSE(filter.inScope(query6, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+
+ // Simulate failover case whereby the standby server detects a
+ // failure of the active server.
+ filter.serveFailoverScopes();
+
+ // The server2 now handles the traffic normally handled by the
+ // server1.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query6, scope_class));
+ EXPECT_EQ("HA_server1", scope_class);
+ EXPECT_NE(scope_class, "HA_server2");
+}
+
+void
+QueryFilterTest::hotStandbyThisBackup6() {
+ HAConfigPtr config = createValidConfiguration(HAConfig::HOT_STANDBY);
+ config->getPeerConfig("server2")->setRole("standby");
+ config->setThisServerName("server3");
+
+ QueryFilter filter(config);
+
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+
+ // By default the backup server doesn't process any traffic.
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ std::string scope_class;
+
+ EXPECT_FALSE(filter.inScope(query6, scope_class));
+
+ // Simulate failover. Although, backup server never starts handling
+ // other server's traffic automatically, it can be manually instructed
+ // to do so. This simulates such scenario.
+ filter.serveFailoverScopes();
+
+ // The backup server now handles the entire traffic, i.e. the traffic
+ // that the primary server handles.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ EXPECT_TRUE(filter.inScope(query6, scope_class));
+}
+
+void
+QueryFilterTest::explicitlyServeScopes() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // Initially, the scopes should be set according to the load
+ // balancing configuration.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Enable "server2" scope.
+ ASSERT_NO_THROW(filter.serveScope("server2"));
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Enable only "server2" scope.
+ ASSERT_NO_THROW(filter.serveScopeOnly("server2"));
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_TRUE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Explicitly enable selected scopes.
+ ASSERT_NO_THROW(filter.serveScopes({ "server1", "server3" }));
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_TRUE(filter.amServingScope("server3"));
+
+ // Revert to defaults.
+ ASSERT_NO_THROW(filter.serveDefaultScopes());
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Disable all scopes.
+ ASSERT_NO_THROW(filter.serveNoScopes());
+ EXPECT_FALSE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Test negative cases.
+ EXPECT_THROW(filter.serveScope("unsupported"), BadValue);
+ EXPECT_THROW(filter.serveScopeOnly("unsupported"), BadValue);
+ EXPECT_THROW(filter.serveScopes({ "server1", "unsupported" }), BadValue);
+}
+
+void
+QueryFilterTest::loadBalancingHaTypes4() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // By default the server1 should serve its own scope only. The
+ // server2 should serve its scope.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Use DHCPDISCOVER to find MAC addresses in scope for server1 and server2.
+ Pkt4Ptr server1_pkt;
+ Pkt4Ptr server2_pkt;
+ const unsigned max_scope_tries = 100;
+ for (unsigned i = 0; i < max_scope_tries; ++i) {
+ // Create query with random HW address.
+ std::string scope_class;
+ Pkt4Ptr query4 = createQuery4(randomKey(HWAddr::ETHERNET_HWADDR_LEN));
+ // If the query is in scope then we're done.
+ if (filter.inScope(query4, scope_class)) {
+ ASSERT_EQ("HA_server1", scope_class);
+ server1_pkt = query4;
+ if (server2_pkt) {
+ break;
+ }
+ } else {
+ ASSERT_EQ("HA_server2", scope_class);
+ server2_pkt = query4;
+ if (server1_pkt) {
+ break;
+ }
+ }
+ }
+
+ ASSERT_TRUE(server1_pkt && server2_pkt) << "do not have both scopes in "
+ << max_scope_tries << ", load balance broken?";
+
+ // Iterate over message types. While setting message type to zero is
+ // semantically invalid, it is useful for testing here. Similarly we exceed
+ // DHCP_TYPES_EOF just to be sure.
+ for (uint8_t msg_type = 0; msg_type < DHCP_TYPES_EOF + 2; ++msg_type) {
+ // All message types should be in scope for server1.
+ server1_pkt->setType(msg_type);
+
+ std::string scope_class;
+ bool is_in_scope = filter.inScope(server1_pkt, scope_class);
+ ASSERT_EQ("HA_server1", scope_class);
+ EXPECT_TRUE(is_in_scope);
+
+ server2_pkt->setType(msg_type);
+ scope_class.clear();
+ is_in_scope = filter.inScope(server2_pkt, scope_class);
+ switch (msg_type) {
+ case DHCPDISCOVER:
+ case DHCPREQUEST:
+ case DHCPDECLINE:
+ case DHCPRELEASE:
+ case DHCPINFORM:
+ // HA message types should be in scope for server2.
+ ASSERT_EQ("HA_server2", scope_class);
+ EXPECT_FALSE(is_in_scope);
+ break;
+ default:
+ // Non HA message types should be in scope for server1.
+ ASSERT_EQ("HA_server1", scope_class);
+ EXPECT_TRUE(is_in_scope);
+ break;
+ }
+ }
+}
+
+void
+QueryFilterTest::loadBalancingHaTypes6() {
+ HAConfigPtr config = createValidConfiguration();
+
+ QueryFilter filter(config);
+
+ // By default the server1 should serve its own scope only. The
+ // server2 should serve its scope.
+ EXPECT_TRUE(filter.amServingScope("server1"));
+ EXPECT_FALSE(filter.amServingScope("server2"));
+ EXPECT_FALSE(filter.amServingScope("server3"));
+
+ // Use DHCPV6_SOLICIT to find MAC addresses in scope for server1 and server2.
+ Pkt6Ptr server1_pkt;
+ Pkt6Ptr server2_pkt;
+ const unsigned max_scope_tries = 100;
+ for (unsigned i = 0; i < max_scope_tries; ++i) {
+ // Create query with random HW address.
+ std::string scope_class;
+
+ // Create query with random DUID.
+ Pkt6Ptr query6 = createQuery6(randomKey(10));
+ if (filter.inScope(query6, scope_class)) {
+ ASSERT_EQ("HA_server1", scope_class);
+ // In scope for server1, save it.
+ server1_pkt = query6;
+ if (server2_pkt) {
+ // Have both, we're done.
+ break;
+ }
+ } else {
+ ASSERT_EQ("HA_server2", scope_class);
+ // In scope for server2, save it.
+ server2_pkt = query6;
+ if (server1_pkt) {
+ // Have both, we're done.
+ break;
+ }
+ }
+ }
+
+ ASSERT_TRUE(server1_pkt && server2_pkt) << "do not have both scopes in "
+ << max_scope_tries << ", load balance broken?";
+
+ // Iterate over message types. While setting message type to zero is
+ // semantically invalid, it is useful for testing here. Similarly we exceed
+ // DHCPV6_TYPES_EOF just to be sure.
+ for (uint8_t msg_type = 0; msg_type < DHCPV6_TYPES_EOF + 2; ++msg_type) {
+ // All message types should be in scope for server1.
+ server1_pkt->setType(msg_type);
+
+ std::string scope_class;
+ bool is_in_scope = filter.inScope(server1_pkt, scope_class);
+ ASSERT_EQ("HA_server1", scope_class);
+ EXPECT_TRUE(is_in_scope);
+
+ server2_pkt->setType(msg_type);
+ scope_class.clear();
+ is_in_scope = filter.inScope(server2_pkt, scope_class);
+ switch (msg_type) {
+ case DHCPV6_SOLICIT:
+ case DHCPV6_REQUEST:
+ case DHCPV6_CONFIRM:
+ case DHCPV6_RENEW:
+ case DHCPV6_REBIND:
+ case DHCPV6_RELEASE:
+ case DHCPV6_DECLINE:
+ // HA message types should be in scope for server2.
+ ASSERT_EQ("HA_server2", scope_class);
+ EXPECT_FALSE(is_in_scope);
+ break;
+ default:
+ // Non HA message types should be in scope for server1.
+ ASSERT_EQ("HA_server1", scope_class);
+ EXPECT_TRUE(is_in_scope);
+ break;
+ }
+ }
+}
+
+TEST_F(QueryFilterTest, loadBalancingClientIdThisPrimary) {
+ loadBalancingClientIdThisPrimary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingClientIdThisPrimaryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingClientIdThisPrimary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisPrimary) {
+ loadBalancingThisPrimary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisPrimaryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisPrimary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisSecondary) {
+ loadBalancingThisSecondary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisSecondaryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisSecondary();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisBackup) {
+ loadBalancingThisBackup();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisBackupMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisBackup();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisPrimary) {
+ hotStandbyThisPrimary();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisPrimaryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisPrimary();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisSecondary) {
+ hotStandbyThisSecondary();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisSecondaryMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisSecondary();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisBackup) {
+ hotStandbyThisBackup();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisBackupMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisBackup();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisPrimary6) {
+ loadBalancingThisPrimary6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisPrimary6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisPrimary6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisSecondary6) {
+ loadBalancingThisSecondary6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisSecondary6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisSecondary6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisBackup6) {
+ loadBalancingThisBackup6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingThisBackup6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingThisBackup6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisPrimary6) {
+ hotStandbyThisPrimary6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisPrimary6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisPrimary6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisSecondary6) {
+ hotStandbyThisSecondary6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisSecondary6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisSecondary6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisBackup6) {
+ hotStandbyThisBackup6();
+}
+
+TEST_F(QueryFilterTest, hotStandbyThisBackup6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ hotStandbyThisBackup6();
+}
+
+TEST_F(QueryFilterTest, explicitlyServeScopes) {
+ explicitlyServeScopes();
+}
+
+TEST_F(QueryFilterTest, explicitlyServeScopesMultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ explicitlyServeScopes();
+}
+
+TEST_F(QueryFilterTest, loadBalancingHaTypes4) {
+ loadBalancingHaTypes4();
+}
+
+TEST_F(QueryFilterTest, loadBalancingHaTypes4MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingHaTypes4();
+}
+
+TEST_F(QueryFilterTest, loadBalancingHaTypes6) {
+ loadBalancingHaTypes6();
+}
+
+TEST_F(QueryFilterTest, loadBalancingHaTypes6MultiThreading) {
+ MultiThreadingMgr::instance().setMode(true);
+ loadBalancingHaTypes6();
+}
+
+}
diff --git a/src/hooks/dhcp/high_availability/tests/run_unittests.cc b/src/hooks/dhcp/high_availability/tests/run_unittests.cc
new file mode 100644
index 0000000..3b1baf9
--- /dev/null
+++ b/src/hooks/dhcp/high_availability/tests/run_unittests.cc
@@ -0,0 +1,19 @@
+// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+
+#include <log/logger_support.h>
+#include <gtest/gtest.h>
+
+int
+main(int argc, char* argv[]) {
+ ::testing::InitGoogleTest(&argc, argv);
+ isc::log::initLogger();
+ int result = RUN_ALL_TESTS();
+
+ return (result);
+}