path: root/bin/tests/system/dnstap/tests.sh

#!/bin/sh

# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0.  If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.

set -e

. ../conf.sh

DIGOPTS="+short -p ${PORT}"
RNDCCMD="$RNDC -p ${CONTROLPORT} -c ../common/rndc.conf"

status=0

# dnstap_data_ready <fstrm_capture_PID> <capture_file> <min_file_size>
# Flushes capture_file and checks wheter its size is >= min_file_size.
dnstap_data_ready() {
	# Process id of running fstrm_capture.
	fstrm_capture_pid=$1
	# Output file provided to fstrm_capture via -w switch.
	capture_file=$2
	# Minimum expected file size.
	min_size_expected=$3

	kill -HUP $fstrm_capture_pid
	file_size=$(wc -c < "$capture_file" | tr -d ' ')
	if [ $file_size -lt $min_size_expected ]; then
		return 1
	fi
}


for bad in bad-*.conf
do
	ret=0
	echo_i "checking that named-checkconf detects error in $bad"
	{ $CHECKCONF $bad > /dev/null 2>&1; rc=$?; } || true
	if [ $rc != 1 ]; then echo_i "failed"; ret=1; fi
	status=$((status + ret))
done

for good in good-*.conf
do
	ret=0
	echo_i "checking that named-checkconf detects no error in $good"
	{ $CHECKCONF $good > /dev/null 2>&1; rc=$?; } || true
	if [ $rc != 0 ]; then echo_i "failed"; ret=1; fi
	status=$((status + ret))
done

echo_i "wait for servers to finish loading"
ret=0
wait_for_log 20 "all zones loaded" ns1/named.run || ret=1
wait_for_log 20 "all zones loaded" ns2/named.run || ret=1
wait_for_log 20 "all zones loaded" ns3/named.run || ret=1
wait_for_log 20 "all zones loaded" ns4/named.run || ret=1
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

# both the 'a.example/A' lookup and the './NS' lookup to ns1
# need to complete before reopening/rolling for the counts to
# be correct.

echo_i "prime cache"
ret=0
$DIG $DIGOPTS @10.53.0.3 a.example > dig.out || true
wait_for_log 20 "(.): reset client" ns1/named.run || true
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

# check three different dnstap reopen/roll methods:
# ns1: dnstap-reopen; ns2: dnstap -reopen; ns3: dnstap -roll
mv ns1/dnstap.out ns1/dnstap.out.save
mv ns2/dnstap.out ns2/dnstap.out.save

if [ -n "$FSTRM_CAPTURE" ] ; then
	ret=0
	echo_i "starting fstrm_capture"
	$FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \
		-w dnstap.out > fstrm_capture.out.1 2>&1 &
	fstrm_capture_pid=$!
	wait_for_log 10 "socket path ns4/dnstap.out" fstrm_capture.out.1 || ret=1
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))
fi

echo_i "reopen/roll capture streams"
ret=0
$RNDCCMD -s 10.53.0.1 dnstap-reopen | sed 's/^/ns1 /' | cat_i
$RNDCCMD -s 10.53.0.2 dnstap -reopen | sed 's/^/ns2 /' | cat_i
$RNDCCMD -s 10.53.0.3 dnstap -roll | sed 's/^/ns3 /' | cat_i
$RNDCCMD -s 10.53.0.4 dnstap -reopen | sed 's/^/ns4 /' | cat_i

echo_i "send test traffic"
ret=0
$DIG $DIGOPTS @10.53.0.3 a.example > dig.out || ret=1

# send an UPDATE to ns2
$NSUPDATE <<- EOF
server 10.53.0.2 ${PORT}
zone example
update add b.example 3600 in a 10.10.10.10
send
EOF

# XXX: file output should be flushed once a second according
# to the libfstrm source, but it doesn't seem to happen until
# enough data has accumulated. to get all the output, we stop
# the name servers, forcing a flush on shutdown. it would be
# nice to find a better way to do this.
$RNDCCMD -s 10.53.0.1 stop | sed 's/^/ns1 /' | cat_i
$RNDCCMD -s 10.53.0.2 stop | sed 's/^/ns2 /' | cat_i
$RNDCCMD -s 10.53.0.3 stop | sed 's/^/ns3 /' | cat_i

sleep 1

echo_i "checking initial message counts"

udp1=$($DNSTAPREAD ns1/dnstap.out.save | grep "UDP " | wc -l)
tcp1=$($DNSTAPREAD ns1/dnstap.out.save | grep "TCP " | wc -l)
aq1=$($DNSTAPREAD ns1/dnstap.out.save | grep "AQ " | wc -l)
ar1=$($DNSTAPREAD ns1/dnstap.out.save | grep "AR " | wc -l)
cq1=$($DNSTAPREAD ns1/dnstap.out.save | grep "CQ " | wc -l)
cr1=$($DNSTAPREAD ns1/dnstap.out.save | grep "CR " | wc -l)
rq1=$($DNSTAPREAD ns1/dnstap.out.save | grep "RQ " | wc -l)
rr1=$($DNSTAPREAD ns1/dnstap.out.save | grep "RR " | wc -l)
uq1=$($DNSTAPREAD ns1/dnstap.out.save | grep "UQ " | wc -l)
ur1=$($DNSTAPREAD ns1/dnstap.out.save | grep "UR " | wc -l)

udp2=$($DNSTAPREAD ns2/dnstap.out.save | grep "UDP " | wc -l)
tcp2=$($DNSTAPREAD ns2/dnstap.out.save | grep "TCP " | wc -l)
aq2=$($DNSTAPREAD ns2/dnstap.out.save | grep "AQ " | wc -l)
ar2=$($DNSTAPREAD ns2/dnstap.out.save | grep "AR " | wc -l)
cq2=$($DNSTAPREAD ns2/dnstap.out.save | grep "CQ " | wc -l)
cr2=$($DNSTAPREAD ns2/dnstap.out.save | grep "CR " | wc -l)
rq2=$($DNSTAPREAD ns2/dnstap.out.save | grep "RQ " | wc -l)
rr2=$($DNSTAPREAD ns2/dnstap.out.save | grep "RR " | wc -l)
uq2=$($DNSTAPREAD ns2/dnstap.out.save | grep "UQ " | wc -l)
ur2=$($DNSTAPREAD ns2/dnstap.out.save | grep "UR " | wc -l)

mv ns3/dnstap.out.0 ns3/dnstap.out.save
udp3=$($DNSTAPREAD ns3/dnstap.out.save | grep "UDP " | wc -l)
tcp3=$($DNSTAPREAD ns3/dnstap.out.save | grep "TCP " | wc -l)
aq3=$($DNSTAPREAD ns3/dnstap.out.save | grep "AQ " | wc -l)
ar3=$($DNSTAPREAD ns3/dnstap.out.save | grep "AR " | wc -l)
cq3=$($DNSTAPREAD ns3/dnstap.out.save | grep "CQ " | wc -l)
cr3=$($DNSTAPREAD ns3/dnstap.out.save | grep "CR " | wc -l)
rq3=$($DNSTAPREAD ns3/dnstap.out.save | grep "RQ " | wc -l)
rr3=$($DNSTAPREAD ns3/dnstap.out.save | grep "RR " | wc -l)
uq3=$($DNSTAPREAD ns3/dnstap.out.save | grep "UQ " | wc -l)
ur3=$($DNSTAPREAD ns3/dnstap.out.save | grep "UR " | wc -l)

echo_i "checking UDP message counts"
ret=0
[ $udp1 -eq 0 ] || {
	echo_i "ns1 $udp1 expected 0"
	ret=1
}
[ $udp2 -eq 2 ] || {
	echo_i "ns2 $udp2 expected 2"
	ret=1
}
[ $udp3 -eq 4 ] || {
	echo_i "ns3 $udp3 expected 4"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking TCP message counts"
ret=0
[ $tcp1 -eq 6 ] || {
	echo_i "ns1 $tcp1 expected 6"
	ret=1
}
[ $tcp2 -eq 2 ] || {
	echo_i "ns2 $tcp2 expected 2"
	ret=1
}
[ $tcp3 -eq 6 ] || {
	echo_i "ns3 $tcp3 expected 6"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking AUTH_QUERY message counts"
ret=0
[ $aq1 -eq 3 ] || {
	echo_i "ns1 $aq1 exepcted 3"
	ret=1
}
[ $aq2 -eq 2 ] || {
	echo_i "ns2 $aq2 expected 2"
	ret=1
}
[ $aq3 -eq 1 ] || {
	echo_i "ns3 $aq3 expected 1"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking AUTH_RESPONSE message counts"
ret=0
[ $ar1 -eq 2 ] || {
	echo_i "ns1 $ar1 expected 2"
	ret=1
}
[ $ar2 -eq 1 ] || {
	echo_i "ns2 $ar2 expected 1"
	ret=1
}
[ $ar3 -eq 0 ] || {
	echo_i "ns3 $ar3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking CLIENT_QUERY message counts"
ret=0
[ $cq1 -eq 0 ] || {
	echo_i "ns1 $cq1 expected 0"
	ret=1
}
[ $cq2 -eq 0 ] || {
	echo_i "ns2 $cq2 expected 0"
	ret=1
}
[ $cq3 -eq 1 ] || {
	echo_i "ns3 $cq3 expected 1"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking CLIENT_RESPONSE message counts"
ret=0
[ $cr1 -eq 1 ] || {
	echo_i "ns1 $cr1 expected 1"
	ret=1
}
[ $cr2 -eq 1 ] || {
	echo_i "ns2 $cr2 expected 1"
	ret=1
}
[ $cr3 -eq 2 ] || {
	echo_i "ns3 $cr3 expected 2"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking RESOLVER_QUERY message counts"
ret=0
[ $rq1 -eq 0 ] || {
	echo_i "ns1 $rq1 expected 0"
	ret=1
}
[ $rq2 -eq 0 ] || {
	echo_i "ns2 $rq2 expected 0"
	ret=1
}
[ $rq3 -eq 3 ] || {
	echo_i "ns3 $rq3 expected 3"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking RESOLVER_RESPONSE message counts"
ret=0
[ $rr1 -eq 0 ] || {
	echo_i "ns1 $rr1 expected 0"
	ret=1
}
[ $rr2 -eq 0 ] || {
	echo_i "ns2 $rr2 expected 0"
	ret=1
}
[ $rr3 -eq 3 ] || {
	echo_i "ns3 $rr3 expected 3"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking UPDATE_QUERY message counts"
ret=0
[ $uq1 -eq 0 ] || {
	echo_i "ns1 $uq1 expected 0"
	ret=1
}
[ $uq2 -eq 0 ] || {
	echo_i "ns2 $uq2 expected 0"
	ret=1
}
[ $uq3 -eq 0 ] || {
	echo_i "ns3 $uq3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking UPDATE_RESPONSE message counts"
ret=0
[ $ur1 -eq 0 ] || {
	echo_i "ns1 $ur1 expected 0"
	ret=1
}
[ $ur2 -eq 0 ] || {
	echo_i "ns2 $ur2 expected 0"
	ret=1
}
[ $ur3 -eq 0 ] || {
	echo_i "ns3 $ur3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking reopened message counts"

udp1=$($DNSTAPREAD ns1/dnstap.out | grep "UDP " | wc -l)
tcp1=$($DNSTAPREAD ns1/dnstap.out | grep "TCP " | wc -l)
aq1=$($DNSTAPREAD ns1/dnstap.out | grep "AQ " | wc -l)
ar1=$($DNSTAPREAD ns1/dnstap.out | grep "AR " | wc -l)
cq1=$($DNSTAPREAD ns1/dnstap.out | grep "CQ " | wc -l)
cr1=$($DNSTAPREAD ns1/dnstap.out | grep "CR " | wc -l)
rq1=$($DNSTAPREAD ns1/dnstap.out | grep "RQ " | wc -l)
rr1=$($DNSTAPREAD ns1/dnstap.out | grep "RR " | wc -l)
uq1=$($DNSTAPREAD ns1/dnstap.out | grep "UQ " | wc -l)
ur1=$($DNSTAPREAD ns1/dnstap.out | grep "UR " | wc -l)

udp2=$($DNSTAPREAD ns2/dnstap.out | grep "UDP " | wc -l)
tcp2=$($DNSTAPREAD ns2/dnstap.out | grep "TCP " | wc -l)
aq2=$($DNSTAPREAD ns2/dnstap.out | grep "AQ " | wc -l)
ar2=$($DNSTAPREAD ns2/dnstap.out | grep "AR " | wc -l)
cq2=$($DNSTAPREAD ns2/dnstap.out | grep "CQ " | wc -l)
cr2=$($DNSTAPREAD ns2/dnstap.out | grep "CR " | wc -l)
rq2=$($DNSTAPREAD ns2/dnstap.out | grep "RQ " | wc -l)
rr2=$($DNSTAPREAD ns2/dnstap.out | grep "RR " | wc -l)
uq2=$($DNSTAPREAD ns2/dnstap.out | grep "UQ " | wc -l)
ur2=$($DNSTAPREAD ns2/dnstap.out | grep "UR " | wc -l)

udp3=$($DNSTAPREAD ns3/dnstap.out | grep "UDP " | wc -l)
tcp3=$($DNSTAPREAD ns3/dnstap.out | grep "TCP " | wc -l)
aq3=$($DNSTAPREAD ns3/dnstap.out | grep "AQ " | wc -l)
ar3=$($DNSTAPREAD ns3/dnstap.out | grep "AR " | wc -l)
cq3=$($DNSTAPREAD ns3/dnstap.out | grep "CQ " | wc -l)
cr3=$($DNSTAPREAD ns3/dnstap.out | grep "CR " | wc -l)
rq3=$($DNSTAPREAD ns3/dnstap.out | grep "RQ " | wc -l)
rr3=$($DNSTAPREAD ns3/dnstap.out | grep "RR " | wc -l)
uq3=$($DNSTAPREAD ns3/dnstap.out | grep "UQ " | wc -l)
ur3=$($DNSTAPREAD ns3/dnstap.out | grep "UR " | wc -l)

echo_i "checking UDP message counts"
ret=0
[ $udp1 -eq 0 ] || {
	echo_i "ns1 $udp1 expected 0"
	ret=1
}
[ $udp2 -eq 2 ] || {
	echo_i "ns2 $udp2 expected 2"
	ret=1
}
[ $udp3 -eq 2 ] || {
	echo_i "ns3 $udp3 expected 2"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking TCP message counts"
ret=0
[ $tcp1 -eq 0 ] || {
	echo_i "ns1 $tcp1 expected 0"
	ret=1
}
[ $tcp2 -eq 0 ] || {
	echo_i "ns2 $tcp2 expected 0"
	ret=1
}
[ $tcp3 -eq 0 ] || {
	echo_i "ns3 $tcp3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking AUTH_QUERY message counts"
ret=0
[ $aq1 -eq 0 ] || {
	echo_i "ns1 $aq1 exepcted 0"
	ret=1
}
[ $aq2 -eq 0 ] || {
	echo_i "ns2 $aq2 expected 0"
	ret=1
}
[ $aq3 -eq 0 ] || {
	echo_i "ns3 $aq3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking AUTH_RESPONSE message counts"
ret=0
[ $ar1 -eq 0 ] || {
	echo_i "ns1 $ar1 expected 0"
	ret=1
}
[ $ar2 -eq 0 ] || {
	echo_i "ns2 $ar2 expected 0"
	ret=1
}
[ $ar3 -eq 0 ] || {
	echo_i "ns3 $ar3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking CLIENT_QUERY message counts"
ret=0
[ $cq1 -eq 0 ] || {
	echo_i "ns1 $cq1 expected 0"
	ret=1
}
[ $cq2 -eq 0 ] || {
	echo_i "ns2 $cq2 expected 0"
	ret=1
}
[ $cq3 -eq 1 ] || {
	echo_i "ns3 $cq3 expected 1"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking CLIENT_RESPONSE message counts"
ret=0
[ $cr1 -eq 0 ] || {
	echo_i "ns1 $cr1 expected 0"
	ret=1
}
[ $cr2 -eq 0 ] || {
	echo_i "ns2 $cr2 expected 0"
	ret=1
}
[ $cr3 -eq 1 ] || {
	echo_i "ns3 $cr3 expected 1"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking RESOLVER_QUERY message counts"
ret=0
[ $rq1 -eq 0 ] || {
	echo_i "ns1 $rq1 expected 0"
	ret=1
}
[ $rq2 -eq 0 ] || {
	echo_i "ns2 $rq2 expected 0"
	ret=1
}
[ $rq3 -eq 0 ] || {
	echo_i "ns3 $rq3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking RESOLVER_RESPONSE message counts"
ret=0
[ $rr1 -eq 0 ] || {
	echo_i "ns1 $rr1 expected 0"
	ret=1
}
[ $rr2 -eq 0 ] || {
	echo_i "ns2 $rr2 expected 0"
	ret=1
}
[ $rr3 -eq 0 ] || {
	echo_i "ns3 $rr3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking UPDATE_QUERY message counts"
ret=0
[ $uq1 -eq 0 ] || {
	echo_i "ns1 $uq1 expected 0"
	ret=1
}
[ $uq2 -eq 1 ] || {
	echo_i "ns2 $uq2 expected 1"
	ret=1
}
[ $uq3 -eq 0 ] || {
	echo_i "ns3 $uq3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking UPDATE_RESPONSE message counts"
ret=0
[ $ur1 -eq 0 ] || {
	echo_i "ns1 $ur1 expected 0"
	ret=1
}
[ $ur2 -eq 1 ] || {
	echo_i "ns2 $ur2 expected 1"
	ret=1
}
[ $ur3 -eq 0 ] || {
	echo_i "ns3 $ur3 expected 0"
	ret=1
}
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

echo_i "checking whether destination UDP port is logged for client queries"
ret=0
$DNSTAPREAD ns3/dnstap.out.save | grep -Eq "CQ [0-9:.]+ -> 10.53.0.3:${PORT} UDP" || ret=1
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

HAS_PYYAML=0
if [ -x "$PYTHON" ] ; then
	$PYTHON -c "import yaml" 2> /dev/null && HAS_PYYAML=1
fi

if [ $HAS_PYYAML -ne 0 ] ; then
	echo_i "checking dnstap-read YAML output"
	ret=0
        {
	  $PYTHON ydump.py "$DNSTAPREAD" "ns3/dnstap.out.save" > ydump.out || ret=1
        } | cat_i
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))
fi

echo_i "checking dnstap-read hex output"
ret=0
hex=$($DNSTAPREAD -x ns3/dnstap.out | tail -1)
echo $hex | $WIRETEST > dnstap.hex
grep 'status: NOERROR' dnstap.hex > /dev/null 2>&1 || ret=1
grep 'ANSWER: 3, AUTHORITY: 1' dnstap.hex > /dev/null 2>&1 || ret=1
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

if [ -n "$FSTRM_CAPTURE" ] ; then
	$DIG $DIGOPTS @10.53.0.4 a.example > dig.out

	# send an UPDATE to ns4
	$NSUPDATE <<- EOF > nsupdate.out 2>&1 && ret=1
	server 10.53.0.4 ${PORT}
	zone example
	update add b.example 3600 in a 10.10.10.10
	send
EOF
	grep "update failed: NOTAUTH" nsupdate.out > /dev/null || ret=1

	echo_i "checking unix socket message counts"
	sleep 2
	retry_quiet 5 dnstap_data_ready $fstrm_capture_pid dnstap.out 450 || {
		echo_i "dnstap output file smaller than expected"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))
	kill $fstrm_capture_pid
	wait
	udp4=$($DNSTAPREAD dnstap.out | grep "UDP " | wc -l)
	tcp4=$($DNSTAPREAD dnstap.out | grep "TCP " | wc -l)
	aq4=$($DNSTAPREAD dnstap.out | grep "AQ " | wc -l)
	ar4=$($DNSTAPREAD dnstap.out | grep "AR " | wc -l)
	cq4=$($DNSTAPREAD dnstap.out | grep "CQ " | wc -l)
	cr4=$($DNSTAPREAD dnstap.out | grep "CR " | wc -l)
	rq4=$($DNSTAPREAD dnstap.out | grep "RQ " | wc -l)
	rr4=$($DNSTAPREAD dnstap.out | grep "RR " | wc -l)
	uq4=$($DNSTAPREAD dnstap.out | grep "UQ " | wc -l)
	ur4=$($DNSTAPREAD dnstap.out | grep "UR " | wc -l)

	echo_i "checking UDP message counts"
	ret=0
	[ $udp4 -eq 4 ] || {
		echo_i "ns4 $udp4 expected 4"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking TCP message counts"
	ret=0
	[ $tcp4 -eq 0 ] || {
		echo_i "ns4 $tcp4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking AUTH_QUERY message counts"
	ret=0
	[ $aq4 -eq 0 ] || {
		echo_i "ns4 $aq4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking AUTH_RESPONSE message counts"
	ret=0
	[ $ar4 -eq 0 ] || {
		echo_i "ns4 $ar4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking CLIENT_QUERY message counts"
	ret=0
	[ $cq4 -eq 1 ] || {
		echo_i "ns4 $cq4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking CLIENT_RESPONSE message counts"
	ret=0
	[ $cr4 -eq 1 ] || {
		echo_i "ns4 $cr4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking RESOLVER_QUERY message counts"
	ret=0
	[ $rq4 -eq 0 ] || {
		echo_i "ns4 $rq4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking RESOLVER_RESPONSE message counts"
	ret=0
	[ $rr4 -eq 0 ] || {
		echo_i "ns4 $rr4 expected 0"
		ret=1
	}

	echo_i "checking UPDATE_QUERY message counts"
	ret=0
	[ $uq4 -eq 1 ] || {
		echo_i "ns4 $uq4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking UPDATE_RESPONSE message counts"
	ret=0
	[ $ur4 -eq 1 ] || {
		echo_i "ns4 $ur4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	mv dnstap.out dnstap.out.save

	echo_i "restarting fstrm_capture"
	$FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \
		-w dnstap.out > fstrm_capture.out.2 2>&1 &
	fstrm_capture_pid=$!
	wait_for_log 10 "socket path ns4/dnstap.out" fstrm_capture.out.2 || {
		echo_i "failed"
		ret=1
	}
	$RNDCCMD -s 10.53.0.4 dnstap -reopen | sed 's/^/ns4 /' | cat_i
	$DIG $DIGOPTS @10.53.0.4 a.example > dig.out

	echo_i "checking reopened unix socket message counts"
	sleep 2
	retry_quiet 5 dnstap_data_ready $fstrm_capture_pid dnstap.out 270 || {
		echo_i "dnstap output file smaller than expected"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))
	kill $fstrm_capture_pid
	wait
	udp4=$($DNSTAPREAD dnstap.out | grep "UDP " | wc -l)
	tcp4=$($DNSTAPREAD dnstap.out | grep "TCP " | wc -l)
	aq4=$($DNSTAPREAD dnstap.out | grep "AQ " | wc -l)
	ar4=$($DNSTAPREAD dnstap.out | grep "AR " | wc -l)
	cq4=$($DNSTAPREAD dnstap.out | grep "CQ " | wc -l)
	cr4=$($DNSTAPREAD dnstap.out | grep "CR " | wc -l)
	rq4=$($DNSTAPREAD dnstap.out | grep "RQ " | wc -l)
	rr4=$($DNSTAPREAD dnstap.out | grep "RR " | wc -l)
	uq4=$($DNSTAPREAD dnstap.out | grep "UQ " | wc -l)
	ur4=$($DNSTAPREAD dnstap.out | grep "UR " | wc -l)

	echo_i "checking UDP message counts"
	ret=0
	[ $udp4 -eq 2 ] || {
		echo_i "ns4 $udp4 expected 2"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking TCP message counts"
	ret=0
	[ $tcp4 -eq 0 ] || {
		echo_i "ns4 $tcp4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking AUTH_QUERY message counts"
	ret=0
	[ $aq4 -eq 0 ] || {
		echo_i "ns4 $aq4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking AUTH_RESPONSE message counts"
	ret=0
	[ $ar4 -eq 0 ] || {
		echo_i "ns4 $ar4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking CLIENT_QUERY message counts"
	ret=0
	[ $cq4 -eq 1 ] || {
		echo_i "ns4 $cq4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking CLIENT_RESPONSE message counts"
	ret=0
	[ $cr4 -eq 1 ] || {
		echo_i "ns4 $cr4 expected 1"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking RESOLVER_QUERY message counts"
	ret=0
	[ $rq4 -eq 0 ] || {
		echo_i "ns4 $rq4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking RESOLVER_RESPONSE message counts"
	ret=0
	[ $rr4 -eq 0 ] || {
		echo_i "ns4 $rr4 expected 0"
		ret=1
	}

	echo_i "checking UPDATE_QUERY message counts"
	ret=0
	[ $uq4 -eq 0 ] || {
		echo_i "ns4 $uq4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))

	echo_i "checking UPDATE_RESPONSE message counts"
	ret=0
	[ $ur4 -eq 0 ] || {
		echo_i "ns4 $ur4 expected 0"
		ret=1
	}
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status + ret))
fi

echo_i "checking large packet printing"
ret=0
# Expect one occurrence of "opcode: QUERY" below "reponse_message_data" and
# another one below "response_message".
lines=$($DNSTAPREAD -y large-answer.fstrm | grep -c "opcode: QUERY")
[ $lines -eq 2 ] || ret=1
if [ $ret != 0 ]; then echo_i "failed"; fi
status=$((status + ret))

_test_dnstap_roll() (
    ip="$1"
    ns="$2"
    n="$3"

    $RNDCCMD -s "${ip}" dnstap -roll "${n}" | sed "s/^/${ns} /" | cat_i &&
    files=$(find "$ns" -name "dnstap.out.[0-9]" | wc -l) &&
    test "$files" -eq "${n}" && test "$files" -ge "1" || return 1
)


test_dnstap_roll() {
	echo_i "checking 'rndc -roll $4' ($1)"
	ret=0

	try=0
	while test $try -lt 12
	do
		touch "$3/dnstap.out.$try"
		try=$((try + 1))
	done

	_repeat 10 _test_dnstap_roll $2 $3 $4 || ret=1
	if [ $ret != 0 ]; then echo_i "failed"; fi
	status=$((status+ret))
}

start_server --noclean --restart --port "${PORT}" ns3
test_dnstap_roll "no versions" 10.53.0.3 ns3 6
test_dnstap_roll "no versions" 10.53.0.3 ns3 3
test_dnstap_roll "no versions" 10.53.0.3 ns3 1

start_server --noclean --restart --port "${PORT}" ns2
test_dnstap_roll "versions" 10.53.0.2 ns2 6
test_dnstap_roll "versions" 10.53.0.2 ns2 3
test_dnstap_roll "versions" 10.53.0.2 ns2 1

echo_i "exit status: $status"
[ "$status" -eq 0 ] || exit 1