Adding upstream version 1:9.11.5.P4+dfsg.upstream/1%9.11.5.P4+dfsgupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

17 files changed, 1123 insertions, 0 deletions

diff --git a/bin/tests/system/mkeys/README b/bin/tests/system/mkeys/README
new file mode 100644
index 0000000..700e6c2
--- /dev/null
+++ b/bin/tests/system/mkeys/README
@@ -0,0 +1,18 @@
+Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+
+See COPYRIGHT in the source root or http://isc.org/copyright.html for terms.
+
+This is for testing managed-keys, in particular with problems
+with RFC 5011 Automated Updates of DNSSEC Trust Anchors.
+
+ns1 is the root server that offers new KSKs and hosts one record for
+testing. The TTL for the zone's records is 2 seconds.
+
+ns2 is a validator that uses managed-keys.  "-T mkeytimers=2/20/40"
+is used so it will attempt do automated updates frequently. "-T tat=1"
+is used so it will send TAT queries once per second.
+
+ns3 is a validator with a broken key in managed-keys.
+
+ns5 is a validator which is prevented from getting a response from the
+root server, causing key refresh queries to fail.
diff --git a/bin/tests/system/mkeys/clean.sh b/bin/tests/system/mkeys/clean.sh
new file mode 100644
index 0000000..17bd50f
--- /dev/null
+++ b/bin/tests/system/mkeys/clean.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */K* */*.signed */trusted.conf */*.jnl */*.bk
+rm -f dsset-. ns1/dsset-.
+rm -f ns*/named.lock
+rm -f */managed-keys.bind* */named.secroots
+rm -f */managed.conf ns1/managed.key ns1/managed.key.id
+rm -f */named.memstats */named.run */named.run.prev
+rm -f dig.out* delv.out* rndc.out* signer.out*
+rm -f ns1/named.secroots ns1/root.db.signed* ns1/root.db.tmp
+rm -f */named.conf
+rm -f ns5/named.args
diff --git a/bin/tests/system/mkeys/ns1/named1.conf.in b/bin/tests/system/mkeys/ns1/named1.conf.in
new file mode 100644
index 0000000..412556b
--- /dev/null
+++ b/bin/tests/system/mkeys/ns1/named1.conf.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS1
+
+acl allowed {
+	! 10.53.0.5;
+	any;
+};
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation yes;
+	allow-query { allowed; };
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type master;
+	file "root.db.signed";
+	allow-update { any; };
+	auto-dnssec maintain;
+};
diff --git a/bin/tests/system/mkeys/ns1/named2.conf.in b/bin/tests/system/mkeys/ns1/named2.conf.in
new file mode 100644
index 0000000..5cb096b
--- /dev/null
+++ b/bin/tests/system/mkeys/ns1/named2.conf.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS1
+
+acl allowed {
+	! 10.53.0.5;
+	any;
+};
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation yes;
+	allow-query { allowed; };
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type master;
+	file "root.db.signed";
+};
diff --git a/bin/tests/system/mkeys/ns1/named3.conf.in b/bin/tests/system/mkeys/ns1/named3.conf.in
new file mode 100644
index 0000000..a0f1dd3
--- /dev/null
+++ b/bin/tests/system/mkeys/ns1/named3.conf.in
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS1
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation yes;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type master;
+	file "root.db.signed";
+};
diff --git a/bin/tests/system/mkeys/ns1/root.db b/bin/tests/system/mkeys/ns1/root.db
new file mode 100644
index 0000000..6ba922a
--- /dev/null
+++ b/bin/tests/system/mkeys/ns1/root.db
@@ -0,0 +1,23 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 20
+. 			IN SOA	gson.nominum.com. a.root.servers.nil. (
+				2000042100   	; serial
+				600         	; refresh
+				600         	; retry
+				1200    	; expire
+				2       	; minimum
+				)
+.			NS	a.root-servers.nil.
+a.root-servers.nil.	A	10.53.0.1
+
+; no delegation
+
+example.		TXT	"This is a test."
diff --git a/bin/tests/system/mkeys/ns1/sign.sh b/bin/tests/system/mkeys/ns1/sign.sh
new file mode 100644
index 0000000..ccc7889
--- /dev/null
+++ b/bin/tests/system/mkeys/ns1/sign.sh
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=../..
+. $SYSTEMTESTTOP/conf.sh
+
+zone=.
+zonefile=root.db
+
+keyname=`$KEYGEN -qfk -r $RANDFILE $zone`
+zskkeyname=`$KEYGEN -q -r $RANDFILE $zone`
+
+$SIGNER -Sg -r $RANDFILE -o $zone $zonefile > /dev/null 2>/dev/null
+
+# Configure the resolving server with a managed trusted key.
+keyfile_to_managed_keys $keyname > managed.conf
+cp managed.conf ../ns2/managed.conf
+cp managed.conf ../ns5/managed.conf
+
+# Configure a trusted key statement (used by delv)
+keyfile_to_trusted_keys $keyname > trusted.conf
+
+#
+#  Save keyname and keyid for managed key id test.
+#
+echo "$keyname" > managed.key
+keyid=`expr $keyname : 'K\.+00.+\([0-9]*\)'`
+keyid=`expr $keyid + 0`
+echo "$keyid" > managed.key.id
diff --git a/bin/tests/system/mkeys/ns2/named.args b/bin/tests/system/mkeys/ns2/named.args
new file mode 100644
index 0000000..5c5a6b2
--- /dev/null
+++ b/bin/tests/system/mkeys/ns2/named.args
@@ -0,0 +1 @@
+-m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -T mkeytimers=5/10/20 -T tat=1
diff --git a/bin/tests/system/mkeys/ns2/named.conf.in b/bin/tests/system/mkeys/ns2/named.conf.in
new file mode 100644
index 0000000..9076243
--- /dev/null
+++ b/bin/tests/system/mkeys/ns2/named.conf.in
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS2
+
+options {
+	query-source address 10.53.0.2;
+	notify-source 10.53.0.2;
+	transfer-source 10.53.0.2;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.2; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation auto;
+	bindkeys-file "managed.conf";
+	servfail-ttl 0;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.2 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "../../common/root.hint";
+};
diff --git a/bin/tests/system/mkeys/ns3/named.args b/bin/tests/system/mkeys/ns3/named.args
new file mode 100644
index 0000000..02f8f67
--- /dev/null
+++ b/bin/tests/system/mkeys/ns3/named.args
@@ -0,0 +1 @@
+-m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -T mkeytimers=5/10/20
diff --git a/bin/tests/system/mkeys/ns3/named.conf.in b/bin/tests/system/mkeys/ns3/named.conf.in
new file mode 100644
index 0000000..0227aba
--- /dev/null
+++ b/bin/tests/system/mkeys/ns3/named.conf.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS3
+
+options {
+	query-source address 10.53.0.3;
+	notify-source 10.53.0.3;
+	transfer-source 10.53.0.3;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.3; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation yes;
+	bindkeys-file "managed.conf";
+	trust-anchor-telemetry no;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "../../common/root.hint";
+};
+
+# purposely broken key for testing
+managed-keys {
+    "." initial-key 257 3 5 "PURPOSELYBROKEN/xs9iVj7QekClcpzjCf0JrvXW1z07hNMqMm6Q2FtIXMbRgfvTtHF3/ZNvcewT9hpfczC+JACHsQSYYdr7UI8oe4nJfal9+2F3pz4a+HR6CqkgrR6WLWQI1Q==";
+};
diff --git a/bin/tests/system/mkeys/ns5/named.conf.in b/bin/tests/system/mkeys/ns5/named.conf.in
new file mode 100644
index 0000000..f2eb35d
--- /dev/null
+++ b/bin/tests/system/mkeys/ns5/named.conf.in
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS5
+
+options {
+	query-source address 10.53.0.5;
+	notify-source 10.53.0.5;
+	transfer-source 10.53.0.5;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.5; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify no;
+	dnssec-enable yes;
+	dnssec-validation auto;
+	bindkeys-file "managed.conf";
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.5 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "../../common/root.hint";
+};
diff --git a/bin/tests/system/mkeys/ns5/named1.args b/bin/tests/system/mkeys/ns5/named1.args
new file mode 100644
index 0000000..efb102a
--- /dev/null
+++ b/bin/tests/system/mkeys/ns5/named1.args
@@ -0,0 +1 @@
+-m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g
diff --git a/bin/tests/system/mkeys/ns5/named2.args b/bin/tests/system/mkeys/ns5/named2.args
new file mode 100644
index 0000000..d222b7f
--- /dev/null
+++ b/bin/tests/system/mkeys/ns5/named2.args
@@ -0,0 +1 @@
+-m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -T mkeytimers=2/20/40
diff --git a/bin/tests/system/mkeys/prereq.sh b/bin/tests/system/mkeys/prereq.sh
new file mode 100644
index 0000000..a0d4e9c
--- /dev/null
+++ b/bin/tests/system/mkeys/prereq.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+exec $SHELL ../testcrypto.sh
diff --git a/bin/tests/system/mkeys/setup.sh b/bin/tests/system/mkeys/setup.sh
new file mode 100644
index 0000000..bd3169f
--- /dev/null
+++ b/bin/tests/system/mkeys/setup.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+$SHELL clean.sh
+
+test -r $RANDFILE || $GENRANDOM 400 $RANDFILE
+
+
+copy_setports ns1/named1.conf.in ns1/named.conf
+copy_setports ns2/named.conf.in ns2/named.conf
+copy_setports ns3/named.conf.in ns3/named.conf
+copy_setports ns5/named.conf.in ns5/named.conf
+
+cp ns5/named1.args ns5/named.args
+
+( cd ns1 && $SHELL sign.sh )
diff --git a/bin/tests/system/mkeys/tests.sh b/bin/tests/system/mkeys/tests.sh
new file mode 100644
index 0000000..f65f49e
--- /dev/null
+++ b/bin/tests/system/mkeys/tests.sh
@@ -0,0 +1,714 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+DIGOPTS="+tcp +noadd +nosea +nostat +nocmd +dnssec -p ${PORT}"
+DELVOPTS="-a ns1/trusted.conf -p ${PORT}"
+RNDCCMD="$RNDC -c $SYSTEMTESTTOP/common/rndc.conf -p ${CONTROLPORT} -s"
+
+wait_for_log() {
+	msg=$1
+	file=$2
+	for i in 1 2 3 4 5 6 7 8 9 10; do
+		nextpart "$file" | grep "$msg" > /dev/null && return
+		sleep 1
+	done
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+	ret=1
+}
+
+mkeys_reconfig_on() {
+	nsidx=$1
+	$RNDCCMD 10.53.0.${nsidx} reconfig . | sed "s/^/ns${nsidx} /" | cat_i
+}
+
+mkeys_reload_on() {
+	nsidx=$1
+	nextpart ns${nsidx}/named.run > /dev/null
+	$RNDCCMD 10.53.0.${nsidx} reload . | sed "s/^/ns${nsidx} /" | cat_i
+	wait_for_log "loaded serial" ns${nsidx}/named.run
+}
+
+mkeys_loadkeys_on() {
+	nsidx=$1
+	nextpart ns${nsidx}/named.run > /dev/null
+	$RNDCCMD 10.53.0.${nsidx} loadkeys . | sed "s/^/ns${nsidx} /" | cat_i
+	wait_for_log "next key event" ns${nsidx}/named.run
+}
+
+mkeys_refresh_on() {
+	nsidx=$1
+	nextpart ns${nsidx}/named.run > /dev/null
+	$RNDCCMD 10.53.0.${nsidx} managed-keys refresh | sed "s/^/ns${nsidx} /" | cat_i
+	wait_for_log "Returned from key fetch in keyfetch_done()" ns${nsidx}/named.run
+}
+
+mkeys_sync_on() {
+	# No race with mkeys_refresh_on() is possible as even if the latter
+	# returns immediately after the expected log message is written, the
+	# managed-keys zone is already locked and the command below calls
+	# dns_zone_flush(), which also attempts to take that zone's lock
+	nsidx=$1
+	$RNDCCMD 10.53.0.${nsidx} managed-keys sync | sed "s/^/ns${nsidx} /" | cat_i
+}
+
+mkeys_status_on() {
+	# No race with mkeys_refresh_on() is possible as even if the latter
+	# returns immediately after the expected log message is written, the
+	# managed-keys zone is already locked and the command below calls
+	# mkey_status(), which in turn calls dns_zone_getrefreshkeytime(),
+	# which also attempts to take that zone's lock
+	nsidx=$1
+	$RNDCCMD 10.53.0.${nsidx} managed-keys status
+}
+
+mkeys_flush_on() {
+	nsidx=$1
+	$RNDCCMD 10.53.0.${nsidx} flush | sed "s/^/ns${nsidx} /" | cat_i
+}
+
+mkeys_secroots_on() {
+	nsidx=$1
+	$RNDCCMD 10.53.0.${nsidx} secroots | sed "s/^/ns${nsidx} /" | cat_i
+}
+
+original=`cat ns1/managed.key`
+originalid=`cat ns1/managed.key.id`
+
+status=0
+n=1
+
+rm -f dig.out.*
+
+echo_i "check for signed record ($n)"
+ret=0
+$DIG $DIGOPTS +norec example.  @10.53.0.1 TXT > dig.out.ns1.test$n || ret=1
+grep "^example\.[ 	]*[0-9].*[ 	]*IN[ 	]*TXT[ 	]*\"This is a test\.\"" dig.out.ns1.test$n > /dev/null || ret=1
+grep "^example\.[ 	]*[0-9].*[ 	]*IN[ 	]*RRSIG[ 	]*TXT[ 	]" dig.out.ns1.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check positive validation with valid trust anchor ($n)"
+ret=0
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+ret=0
+echo_i "check positive validation using delv ($n)"
+$DELV $DELVOPTS @10.53.0.1 txt example > delv.out$n || ret=1
+grep "; fully validated" delv.out$n > /dev/null || ret=1	# redundant
+grep "example..*TXT.*This is a test" delv.out$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" delv.out$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check for failed validation due to wrong key in managed-keys ($n)"
+ret=0
+$DIG $DIGOPTS +noauth example. @10.53.0.3 txt > dig.out.ns3.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns3.test$n > /dev/null && ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns3.test$n > /dev/null && ret=1
+grep "opcode: QUERY, status: SERVFAIL, id" dig.out.ns3.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new trust anchor can be added ($n)"
+ret=0
+standby1=`$KEYGEN -qfk -r $RANDFILE -K ns1 .`
+mkeys_loadkeys_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# there should be two keys listed now
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# one indicates current trust
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# one indicates pending trust
+count=`grep -c "trust pending" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new trust anchor can't be added with bad initial key ($n)"
+ret=0
+mkeys_refresh_on 3
+mkeys_status_on 3 > rndc.out.$n 2>&1
+# there should be one key listed now
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# one line indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# ... and the key is not trusted
+count=`grep -c "no trust" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "remove untrusted standby key, check timer restarts ($n)"
+ret=0
+mkeys_sync_on 2
+t1=`grep "trust pending" ns2/managed-keys.bind`
+$SETTIME -D now -K ns1 $standby1 > /dev/null
+mkeys_loadkeys_on 1
+# Less than a second may have passed since the last time ns2 received a
+# ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
+# timestamp to prevent false negatives caused by the acceptance timer getting
+# reset to the same timestamp.
+sleep 1
+mkeys_refresh_on 2
+mkeys_sync_on 2
+t2=`grep "trust pending" ns2/managed-keys.bind`
+# trust pending date must be different
+[ -n "$t2" ] || ret=1
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+ret=0
+echo_i "restore untrusted standby key, revoke original key ($n)"
+t1=$t2
+$SETTIME -D none -K ns1 $standby1 > /dev/null
+$SETTIME -R now -K ns1 $original > /dev/null
+mkeys_loadkeys_on 1
+# Less than a second may have passed since the last time ns2 received a
+# ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
+# timestamp to prevent false negatives caused by the acceptance timer getting
+# reset to the same timestamp.
+sleep 1
+mkeys_refresh_on 2
+mkeys_sync_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# trust is revoked
+count=`grep -c "trust revoked" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# removal scheduled
+count=`grep -c "remove at" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# trust is still pending on the standby key
+count=`grep -c "trust pending" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# pending date moved forward for the standby key
+t2=`grep "trust pending" ns2/managed-keys.bind`
+[ -n "$t2" ] || ret=1
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+ret=0
+echo_i "refresh managed-keys, ensure same result ($n)"
+t1=$t2
+# Less than a second may have passed since the last time ns2 received a
+# ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
+# timestamp to prevent false negatives caused by the acceptance timer getting
+# reset to the same timestamp.
+sleep 1
+mkeys_refresh_on 2
+mkeys_sync_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# trust is revoked
+count=`grep -c "trust revoked" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# removal scheduled
+count=`grep -c "remove at" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# trust is still pending on the standby key
+count=`grep -c "trust pending" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# pending date moved forward for the standby key
+t2=`grep "trust pending" ns2/managed-keys.bind`
+[ -n "$t2" ] || ret=1
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+ret=0
+echo_i "restore revoked key, ensure same result ($n)"
+t1=$t2
+$SETTIME -R none -D now -K ns1 $original > /dev/null
+mkeys_loadkeys_on 1
+$SETTIME -D none -K ns1 $original > /dev/null
+mkeys_loadkeys_on 1
+# Less than a second may have passed since the last time ns2 received a
+# ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
+# timestamp to prevent false negatives caused by the acceptance timer getting
+# reset to the same timestamp.
+sleep 1
+mkeys_refresh_on 2
+mkeys_sync_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# trust is revoked
+count=`grep -c "trust revoked" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# removal scheduled
+count=`grep -c "remove at" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# trust is still pending on the standby key
+count=`grep -c "trust pending" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+# pending date moved forward for the standby key
+t2=`grep "trust pending" ns2/managed-keys.bind`
+[ -n "$t2" ] || ret=1
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "reinitialize trust anchors"
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc . ns2
+rm -f ns2/managed-keys.bind*
+nextpart ns2/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns2
+
+n=`expr $n + 1`
+echo_i "check that standby key is now trusted ($n)"
+ret=0
+wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# both indicate current trust
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "revoke original key, add new standby ($n)"
+ret=0
+standby2=`$KEYGEN -qfk -r $RANDFILE -K ns1 .`
+$SETTIME -R now -K ns1 $original > /dev/null
+mkeys_loadkeys_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# three keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 3 ] || ret=1
+# one is revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# three lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 3 ] || ret=1
+# one indicates current trust
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# one indicates revoked trust
+count=`grep -c "trust revoked" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# one indicates trust pending
+count=`grep -c "trust pending" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# removal scheduled
+count=`grep -c "remove at" rndc.out.$n`
+[ "$count" -eq 1 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "revoke standby before it is trusted ($n)"
+ret=0
+standby3=`$KEYGEN -qfk -r $RANDFILE -K ns1 .`
+mkeys_loadkeys_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.a.$n 2>&1
+# four keys listed
+count=`grep -c "keyid: " rndc.out.a.$n` 
+[ "$count" -eq 4 ] || { echo "keyid: count ($count) != 4"; ret=1; }
+# one revoked
+count=`grep -c "trust revoked" rndc.out.a.$n` 
+[ "$count" -eq 1 ] || { echo "trust revoked count ($count) != 1"; ret=1; }
+# two pending
+count=`grep -c "trust pending" rndc.out.a.$n` 
+[ "$count" -eq 2 ] || { echo "trust pending count ($count) != 2"; ret=1; }
+$SETTIME -R now -K ns1 $standby3 > /dev/null
+mkeys_loadkeys_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.b.$n 2>&1
+# now three keys listed
+count=`grep -c "keyid: " rndc.out.b.$n` 
+[ "$count" -eq 3 ] || { echo "keyid: count ($count) != 3"; ret=1; }
+# one revoked
+count=`grep -c "trust revoked" rndc.out.b.$n` 
+[ "$count" -eq 1 ] || { echo "trust revoked count ($count) != 1"; ret=1; }
+# one pending
+count=`grep -c "trust pending" rndc.out.b.$n` 
+[ "$count" -eq 1 ] || { echo "trust pending count ($count) != 1"; ret=1; }
+$SETTIME -D now -K ns1 $standby3 > /dev/null
+mkeys_loadkeys_on 1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "wait 20 seconds for key add/remove holddowns to expire ($n)"
+ret=0
+sleep 20
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# none revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 0 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# both indicate current trust
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "revoke all keys, confirm roll to insecure ($n)"
+ret=0
+$SETTIME -D now -K ns1 $original > /dev/null
+$SETTIME -R now -K ns1 $standby1 > /dev/null
+$SETTIME -R now -K ns1 $standby2 > /dev/null
+mkeys_loadkeys_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# two keys listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# both revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# two lines indicating trust status
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# both indicate trust revoked
+count=`grep -c "trust revoked" rndc.out.$n` 
+[ "$count" -eq 2 ] || ret=1
+# both have removal scheduled
+count=`grep -c "remove at" rndc.out.$n`
+[ "$count" -eq 2 ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check for insecure response ($n)"
+ret=0
+mkeys_refresh_on 2
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "status: NOERROR" dig.out.ns2.test$n > /dev/null || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null && ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "reset the root server"
+$SETTIME -D none -R none -K ns1 $original > /dev/null
+$SETTIME -D now -K ns1 $standby1 > /dev/null
+$SETTIME -D now -K ns1 $standby2 > /dev/null
+$SIGNER -Sg -K ns1 -N unixtime -r $RANDFILE -o . ns1/root.db > /dev/null 2>/dev/null
+copy_setports ns1/named2.conf.in ns1/named.conf
+rm -f ns1/root.db.signed.jnl
+mkeys_reconfig_on 1
+
+echo_i "reinitialize trust anchors"
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} . ns2
+rm -f ns2/managed-keys.bind*
+nextpart ns2/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns2
+
+n=`expr $n + 1`
+echo_i "check positive validation ($n)"
+ret=0
+wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "revoke key with bad signature, check revocation is ignored ($n)"
+ret=0
+revoked=`$REVOKE -K ns1 $original`
+rkeyid=`expr $revoked : 'ns1/K\.+00.+0*\([1-9]*[0-9]*[0-9]\)'`
+rm -f ns1/root.db.signed.jnl
+# We need to activate at least one valid DNSKEY to prevent dnssec-signzone from
+# failing.  Alternatively, we could use -P to disable post-sign verification,
+# but we actually do want post-sign verification to happen to ensure the zone
+# is correct before we break it on purpose.
+$SETTIME -R none -D none -K ns1 $standby1 > /dev/null
+$SIGNER -Sg -K ns1 -N unixtime -r $RANDFILE -O full -o . -f signer.out.$n ns1/root.db > /dev/null 2>/dev/null
+cp -f ns1/root.db.signed ns1/root.db.tmp
+BADSIG="SVn2tLDzpNX2rxR4xRceiCsiTqcWNKh7NQ0EQfCrVzp9WEmLw60sQ5kP xGk4FS/xSKfh89hO2O/H20Bzp0lMdtr2tKy8IMdU/mBZxQf2PXhUWRkg V2buVBKugTiOPTJSnaqYCN3rSfV1o7NtC1VNHKKK/D5g6bpDehdn5Gaq kpBhN+MSCCh9OZP2IT20luS1ARXxLlvuSVXJ3JYuuhTsQXUbX/SQpNoB Lo6ahCE55szJnmAxZEbb2KOVnSlZRA6ZBHDhdtO0S4OkvcmTutvcVV+7 w53CbKdaXhirvHIh0mZXmYk2PbPLDY7PU9wSH40UiWPOB9f00wwn6hUe uEQ1Qg=="
+# Less than a second may have passed since ns1 was started.  If we call
+# dnssec-signzone immediately, ns1/root.db.signed will not be reloaded by the
+# subsequent "rndc reload ." call on platforms which do not set the
+# "nanoseconds" field of isc_time_t, due to zone load time being seemingly
+# equal to master file modification time.
+sleep 1
+sed -e "/ $rkeyid \./s, \. .*$, . $BADSIG," signer.out.$n > ns1/root.db.signed
+mkeys_reload_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# one key listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 1 ] || { echo "'keyid:' count ($count) != 1"; ret=1; }
+# it's the original key id
+count=`grep -c "keyid: $originalid" rndc.out.$n` 
+[ "$count" -eq 1 ] || { echo "'keyid: $originalid' count ($count) != 1"; ret=1; }
+# not revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 0 ] || { echo "'REVOKE' count ($count) != 0"; ret=1; }
+# trust is still current
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 1 ] || { echo "'trust' count != 1"; ret=1; }
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 1 ] || { echo "'trusted since' count != 1"; ret=1; }
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check validation fails with bad DNSKEY rrset ($n)"
+ret=0
+mkeys_flush_on 2
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "status: SERVFAIL" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "restore DNSKEY rrset, check validation succeeds again ($n)"
+ret=0
+rm -f ${revoked}.key ${revoked}.private
+rm -f ns1/root.db.signed.jnl
+$SETTIME -D none -R none -K ns1 $original > /dev/null
+$SETTIME -D now -K ns1 $standby1 > /dev/null
+# Less than a second may have passed since ns1 was started.  If we call
+# dnssec-signzone immediately, ns1/root.db.signed will not be reloaded by the
+# subsequent "rndc reload ." call on platforms which do not set the
+# "nanoseconds" field of isc_time_t, due to zone load time being seemingly
+# equal to master file modification time.
+sleep 1
+$SIGNER -Sg -K ns1 -N unixtime -r $RANDFILE -o . ns1/root.db > /dev/null 2>/dev/null
+mkeys_reload_on 1
+mkeys_flush_on 2
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "reset the root server with no keys, check for minimal update ($n)"
+ret=0
+# Refresh keys first to prevent previous checks from influencing this one.
+# Note that we might still get occasional false negatives on some really slow
+# machines, when $t1 equals $t2 due to the time elapsed between "rndc
+# managed-keys status" calls being equal to the normal active refresh period
+# (as calculated per rules listed in RFC 5011 section 2.3) minus an "hour" (as
+# set using -T mkeytimers).
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+t1=`grep 'next refresh:' rndc.out.$n`
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} . ns1
+rm -f ns1/root.db.signed.jnl
+cp ns1/root.db ns1/root.db.signed
+nextpart ns1/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns1
+wait_for_log "loaded serial" ns1/named.run
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# one key listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# it's the original key id
+count=`grep -c "keyid: $originalid" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# not revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 0 ] || ret=1
+# trust is still current
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+t2=`grep 'next refresh:' rndc.out.$n`
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "reset the root server with no signatures, check for minimal update ($n)"
+ret=0
+# Refresh keys first to prevent previous checks from influencing this one
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+t1=`grep 'next refresh:' rndc.out.$n`
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} . ns1
+rm -f ns1/root.db.signed.jnl
+cat ns1/K*.key >> ns1/root.db.signed
+nextpart ns1/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns1
+wait_for_log "loaded serial" ns1/named.run
+# Less than a second may have passed since the last time ns2 received a
+# ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
+# timestamp to prevent minimal update from resetting it to the same timestamp.
+sleep 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+# one key listed
+count=`grep -c "keyid: " rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# it's the original key id
+count=`grep -c "keyid: $originalid" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+# not revoked
+count=`grep -c "REVOKE" rndc.out.$n` 
+[ "$count" -eq 0 ] || ret=1
+# trust is still current
+count=`grep -c "trust" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+count=`grep -c "trusted since" rndc.out.$n` 
+[ "$count" -eq 1 ] || ret=1
+t2=`grep 'next refresh:' rndc.out.$n`
+[ "$t1" = "$t2" ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "restore root server, check validation succeeds again ($n)"
+ret=0
+rm -f ns1/root.db.signed.jnl
+$SIGNER -Sg -K ns1 -N unixtime -r $RANDFILE -o . ns1/root.db > /dev/null 2>/dev/null
+mkeys_reload_on 1
+mkeys_refresh_on 2
+mkeys_status_on 2 > rndc.out.$n 2>&1
+$DIG $DIGOPTS +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check that trust-anchor-telemetry queries are logged ($n)"
+ret=0
+grep "sending trust-anchor-telemetry query '_ta-[0-9a-f]*/NULL" ns2/named.run > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check that trust-anchor-telemetry queries are received ($n)"
+ret=0
+grep "query '_ta-[0-9a-f][0-9a-f]*/NULL/IN' approved" ns1/named.run > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check that trust-anchor-telemetry queries contain the correct key ($n)"
+ret=0
+# convert the hexadecimal key from the TAT query into decimal and
+# compare against the known key.
+tathex=`grep "query '_ta-[0-9a-f][0-9a-f]*/NULL/IN' approved" ns1/named.run | awk '{print $6; exit 0}' | sed -e 's/(_ta-\([0-9a-f][0-9a-f]*\)):/\1/'`
+tatkey=`$PERL -e 'printf("%d\n", hex(@ARGV[0]));' $tathex`
+realkey=`$RNDCCMD 10.53.0.2 secroots - | sed -n 's#.*SHA1/\([0-9][0-9]*\) ; .*managed.*#\1#p'`
+[ "$tatkey" -eq "$realkey" ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check failure to contact root servers does not prevent key refreshes after restart ($n)"
+ret=0
+# By the time we get here, ns5 should have attempted refreshing its managed
+# keys.  These attempts should fail as ns1 is configured to REFUSE all queries
+# from ns5.  Note that named1.args does not contain "-T mkeytimers"; this is to
+# ensure key refresh retry will be scheduled to one actual hour after the first
+# key refresh failure instead of just a few seconds, in order to prevent races
+# between the next scheduled key refresh time and startup time of restarted ns5.
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} . ns5
+nextpart ns5/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns5
+wait_for_log "Returned from key fetch in keyfetch_done()" ns5/named.run
+# ns5/named.run will contain logs from both the old instance and the new
+# instance.  In order for the test to pass, both must attempt a fetch.
+count=`grep -c "Creating key fetch" ns5/named.run`
+[ $count -lt 2 ] && ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check key refreshes are resumed after root servers become available ($n)"
+ret=0
+$PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} . ns5
+# Prevent previous check from affecting this one
+rm -f ns5/managed-keys.bind*
+# named2.args adds "-T mkeytimers=2/20/40" to named1.args as we need to wait for
+# an "hour" until keys are refreshed again after initial failure
+cp ns5/named2.args ns5/named.args
+nextpart ns5/named.run > /dev/null
+$PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} . ns5
+wait_for_log "Returned from key fetch in keyfetch_done() for '.': failure" ns5/named.run
+# ns1 should still REFUSE queries from ns5, so resolving should be impossible
+$DIG $DIGOPTS +noauth example. @10.53.0.5 txt > dig.out.ns5.a.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns5.a.test$n > /dev/null && ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns5.a.test$n > /dev/null && ret=1
+grep "status: SERVFAIL" dig.out.ns5.a.test$n > /dev/null || ret=1
+# Allow queries from ns5 to ns1
+copy_setports ns1/named3.conf.in ns1/named.conf
+rm -f ns1/root.db.signed.jnl
+nextpart ns5/named.run > /dev/null
+mkeys_reconfig_on 1
+wait_for_log "Returned from key fetch in keyfetch_done() for '.': success" ns5/named.run
+# ns1 should not longer REFUSE queries from ns5, so managed keys should be
+# correctly refreshed and resolving should succeed
+$DIG $DIGOPTS +noauth example. @10.53.0.5 txt > dig.out.ns5.b.test$n || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns5.b.test$n > /dev/null || ret=1
+grep "example..*.RRSIG..*TXT" dig.out.ns5.b.test$n > /dev/null || ret=1
+grep "status: NOERROR" dig.out.ns5.b.test$n > /dev/null || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1