14 files changed, 583 insertions, 0 deletions
diff --git a/bin/tests/system/fetchlimit/ans4/ans.pl b/bin/tests/system/fetchlimit/ans4/ans.pl
new file mode 100644
index 0000000..4b14a92
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ans4/ans.pl
@@ -0,0 +1,84 @@
+#!/usr/bin/perl -w
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+#
+# Don't respond if the "norespond" file exists; otherwise respond to
+# any A or AAAA query.
+#
+
+use IO::File;
+use IO::Socket;
+use Net::DNS;
+use Net::DNS::Packet;
+
+my $localport = int($ENV{'PORT'});
+if (!$localport) { $localport = 5300; }
+
+my $sock = IO::Socket::INET->new(LocalAddr => "10.53.0.4",
+   LocalPort => $localport, Proto => "udp") or die "$!";
+
+my $pidf = new IO::File "ans.pid", "w" or die "cannot open pid file: $!";
+print $pidf "$$\n" or die "cannot write pid file: $!";
+$pidf->close or die "cannot close pid file: $!";
+sub rmpid { unlink "ans.pid"; exit 1; };
+
+$SIG{INT} = \&rmpid;
+$SIG{TERM} = \&rmpid;
+
+for (;;) {
+	$sock->recv($buf, 512);
+
+	print "**** request from " , $sock->peerhost, " port ", $sock->peerport, "\n";
+
+	my $packet;
+
+	if ($Net::DNS::VERSION > 0.68) {
+		$packet = new Net::DNS::Packet(\$buf, 0);
+		$@ and die $@;
+	} else {
+		my $err;
+		($packet, $err) = new Net::DNS::Packet(\$buf, 0);
+		$err and die $err;
+	}
+
+	print "REQUEST:\n";
+	$packet->print;
+
+	$packet->header->qr(1);
+
+	my @questions = $packet->question;
+	my $qname = $questions[0]->qname;
+	my $qtype = $questions[0]->qtype;
+
+	my $donotrespond = 0;
+
+	if (-e 'norespond') {
+		$donotrespond = 1;
+	} else {
+		$packet->header->aa(1);
+		if ($qtype eq "A") {
+			$packet->push("answer",
+				      new Net::DNS::RR($qname .
+						       " 300 A 192.0.2.1"));
+		} elsif ($qtype eq "AAAA") {
+			$packet->push("answer",
+				      new Net::DNS::RR($qname .
+						" 300 AAAA 2001:db8:beef::1"));
+		}
+	}
+
+	if ($donotrespond == 0) {
+		$sock->send($packet->data);
+		print "RESPONSE:\n";
+		$packet->print;
+		print "\n";
+	}
+}
diff --git a/bin/tests/system/fetchlimit/clean.sh b/bin/tests/system/fetchlimit/clean.sh
new file mode 100644
index 0000000..9296105
--- /dev/null
+++ b/bin/tests/system/fetchlimit/clean.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.conf */named.memstats */ans.run */named.recursing */named.run
+rm -f dig.out*
+rm -f ans4/norespond
+rm -f ns3/named.stats ns3/named_dump.db
+rm -f burst.input.*
diff --git a/bin/tests/system/fetchlimit/ns1/named.conf.in b/bin/tests/system/fetchlimit/ns1/named.conf.in
new file mode 100644
index 0000000..eabfe0c
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns1/named.conf.in
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify yes;
+};
+
+zone "." {
+	type master;
+	file "root.db";
+};
+
+zone "example.info." {
+	type master;
+	file "example-info.db";
+};
diff --git a/bin/tests/system/fetchlimit/ns1/root.db b/bin/tests/system/fetchlimit/ns1/root.db
new file mode 100644
index 0000000..dab5cea
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns1/root.db
@@ -0,0 +1,22 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 300
+. 			IN SOA	gson.nominum.com. a.root.servers.nil. (
+				2000042100   	; serial
+				600         	; refresh
+				600         	; retry
+				1200    	; expire
+				600       	; minimum
+				)
+.			NS	a.root-servers.nil.
+a.root-servers.nil.	A	10.53.0.1
+
+example.		NS	ns2.example.
+ns2.example.		A	10.53.0.2
diff --git a/bin/tests/system/fetchlimit/ns2/example.db b/bin/tests/system/fetchlimit/ns2/example.db
new file mode 100644
index 0000000..40929b3
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns2/example.db
@@ -0,0 +1,35 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$ORIGIN .
+$TTL 300	; 5 minutes
+example			IN SOA	mname1. . (
+				1	   ; serial
+				20	   ; refresh (20 seconds)
+				20	   ; retry (20 seconds)
+				1814400	   ; expire (3 weeks)
+				3600	   ; minimum (1 hour)
+				)
+example		        NS	ns2.example.
+ns2.example.		A	10.53.0.2
+
+a.example.		A	10.0.0.1
+			MX	10 mail.example.
+
+mail.example.		A	10.0.0.2
+
+lamesub.example.        NS	ns4.example.
+ns4.example.	        A	10.53.0.4
+
+0.example.              A       10.53.1.0
+1.example.              A       10.53.1.1
+2.example.              A       10.53.1.2
+3.example.              A       10.53.1.3
+4.example.              A       10.53.1.4
+5.example.              A       10.53.1.5
diff --git a/bin/tests/system/fetchlimit/ns2/named.conf.in b/bin/tests/system/fetchlimit/ns2/named.conf.in
new file mode 100644
index 0000000..1892756
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns2/named.conf.in
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.2;
+	notify-source 10.53.0.2;
+	transfer-source 10.53.0.2;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.2; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify yes;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.2 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+
+zone "example" {
+	type master;
+	file "example.db";
+	allow-update { any; };
+};
diff --git a/bin/tests/system/fetchlimit/ns3/named.args b/bin/tests/system/fetchlimit/ns3/named.args
new file mode 100644
index 0000000..4f737fc
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns3/named.args
@@ -0,0 +1,2 @@
+# Don't specify '-T clienttest' as it consumes lots of memory with this test
+-D ns3 -X named.lock -m record,size,mctx -c named.conf -d 99 -g -U 4
diff --git a/bin/tests/system/fetchlimit/ns3/named1.conf.in b/bin/tests/system/fetchlimit/ns3/named1.conf.in
new file mode 100644
index 0000000..b470f46
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns3/named1.conf.in
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+controls { /* empty */ };
+
+options {
+	query-source address 10.53.0.3;
+	notify-source 10.53.0.3;
+	transfer-source 10.53.0.3;
+	port @PORT@;
+	directory ".";
+	pid-file "named.pid";
+	listen-on { 10.53.0.3; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify yes;
+	fetches-per-server 400;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "root.hint";
+};
diff --git a/bin/tests/system/fetchlimit/ns3/named2.conf.in b/bin/tests/system/fetchlimit/ns3/named2.conf.in
new file mode 100644
index 0000000..001a958
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns3/named2.conf.in
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.3;
+	notify-source 10.53.0.3;
+	transfer-source 10.53.0.3;
+	port @PORT@;
+	directory ".";
+	pid-file "named.pid";
+	listen-on { 10.53.0.3; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify yes;
+	fetches-per-zone 40;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "root.hint";
+};
diff --git a/bin/tests/system/fetchlimit/ns3/named3.conf.in b/bin/tests/system/fetchlimit/ns3/named3.conf.in
new file mode 100644
index 0000000..0244a9a
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns3/named3.conf.in
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.3;
+	notify-source 10.53.0.3;
+	transfer-source 10.53.0.3;
+	port @PORT@;
+	directory ".";
+	pid-file "named.pid";
+	listen-on { 10.53.0.3; };
+	listen-on-v6 { none; };
+	recursion yes;
+	notify yes;
+	recursive-clients 400;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "." {
+	type hint;
+	file "root.hint";
+};
diff --git a/bin/tests/system/fetchlimit/ns3/root.hint b/bin/tests/system/fetchlimit/ns3/root.hint
new file mode 100644
index 0000000..418ea96
--- /dev/null
+++ b/bin/tests/system/fetchlimit/ns3/root.hint
@@ -0,0 +1,12 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 999999
+.			 IN NS	a.root-servers.nil.
+a.root-servers.nil.	 IN A	10.53.0.1
diff --git a/bin/tests/system/fetchlimit/prereq.sh b/bin/tests/system/fetchlimit/prereq.sh
new file mode 100644
index 0000000..de147a4
--- /dev/null
+++ b/bin/tests/system/fetchlimit/prereq.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+if $PERL -e 'use Net::DNS;' 2>/dev/null
+then
+    :
+else
+    echo "I:This test requires the Net::DNS library." >&2
+    exit 1
+fi
diff --git a/bin/tests/system/fetchlimit/setup.sh b/bin/tests/system/fetchlimit/setup.sh
new file mode 100644
index 0000000..2c83ad1
--- /dev/null
+++ b/bin/tests/system/fetchlimit/setup.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+$SHELL clean.sh 
+
+copy_setports ns1/named.conf.in ns1/named.conf
+copy_setports ns2/named.conf.in ns2/named.conf
+copy_setports ns3/named1.conf.in ns3/named.conf
diff --git a/bin/tests/system/fetchlimit/tests.sh b/bin/tests/system/fetchlimit/tests.sh
new file mode 100644
index 0000000..9250c46
--- /dev/null
+++ b/bin/tests/system/fetchlimit/tests.sh
@@ -0,0 +1,186 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+DIGCMD="$DIG @10.53.0.3 -p ${PORT} +tries=1 +time=1"
+RNDCCMD="$RNDC -p ${CONTROLPORT} -s 10.53.0.3 -c ../common/rndc.conf"
+
+burst() {
+    num=${3:-20}
+    rm -f burst.input.$$
+    while [ $num -gt 0 ]; do
+        num=`expr $num - 1`
+        echo "${num}${1}${2}.lamesub.example A" >> burst.input.$$
+    done
+    $PERL ../ditch.pl -p ${PORT} -s 10.53.0.3 burst.input.$$
+    rm -f burst.input.$$
+}
+
+stat() {
+    clients=`$RNDCCMD status | grep "recursive clients" | 
+            sed 's;.*: \([^/][^/]*\)/.*;\1;'`
+    echo_i "clients: $clients"
+    [ "$clients" = "" ] && return 1
+    [ "$clients" -le $1 ]
+}
+
+status=0
+
+echo_i "checking recursing clients are dropped at the per-server limit"
+ret=0
+# make the server lame and restart
+$RNDCCMD flush
+touch ans4/norespond
+for try in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
+    burst a $try
+    # fetches-per-server is at 400, but at 20qps against a lame server,
+    # we'll reach 200 at the tenth second, and the quota should have been
+    # tuned to less than that by then
+    stat 200 || ret=1
+    [ $ret -eq 1 ] && break
+    sleep 1
+done
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "dumping ADB data"
+$RNDCCMD dumpdb -adb
+info=`grep '10.53.0.4' ns3/named_dump.db | sed 's/.*\(atr [.0-9]*\).*\(quota [0-9]*\).*/\1 \2/'`
+echo_i $info
+set -- $info
+quota=$5
+[ ${5:-200} -lt 200 ] || ret=1
+
+echo_i "checking servfail statistics"
+ret=0
+rm -f ns3/named.stats
+$RNDCCMD stats
+for try in 1 2 3 4 5; do
+    [ -f ns3/named.stats ] && break
+    sleep 1
+done
+sspill=`grep 'spilled due to server' ns3/named.stats | sed 's/\([0-9][0-9]*\) spilled.*/\1/'`
+[ -z "$sspill" ] && sspill=0
+fails=`grep 'queries resulted in SERVFAIL' ns3/named.stats | sed 's/\([0-9][0-9]*\) queries.*/\1/'`
+[ -z "$fails" ] && fails=0
+[ "$fails" -ge "$sspill" ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "checking lame server recovery"
+ret=0
+rm -f ans4/norespond
+for try in 1 2 3 4 5; do
+    burst b $try
+    stat 200 || ret=1
+    [ $ret -eq 1 ] && break
+    sleep 1
+done
+
+echo_i "dumping ADB data"
+$RNDCCMD dumpdb -adb
+info=`grep '10.53.0.4' ns3/named_dump.db | sed 's/.*\(atr [.0-9]*\).*\(quota [0-9]*\).*/\1 \2/'`
+echo_i $info
+set -- $info
+[ ${5:-${quota}} -lt $quota ] || ret=1
+quota=$5
+
+for try in 1 2 3 4 5 6 7 8 9 10; do
+    burst c $try
+    stat 20 || ret=1
+    [ $ret -eq 1 ] && break
+    sleep 1
+done
+
+echo_i "dumping ADB data"
+$RNDCCMD dumpdb -adb
+info=`grep '10.53.0.4' ns3/named_dump.db | sed 's/.*\(atr [.0-9]*\).*\(quota [0-9]*\).*/\1 \2/'`
+echo_i $info
+set -- $info
+[ ${5:-${quota}} -gt $quota ] || ret=1
+quota=$5
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+copy_setports ns3/named2.conf.in ns3/named.conf
+$RNDCCMD reconfig 2>&1 | sed 's/^/ns3 /' | cat_i
+
+echo_i "checking lame server clients are dropped at the per-domain limit"
+ret=0
+fail=0
+success=0
+touch ans4/norespond
+for try in 1 2 3 4 5; do
+    burst b $try 300
+    $DIGCMD a ${try}.example > dig.out.ns3.$try
+    grep "status: NOERROR" dig.out.ns3.$try > /dev/null 2>&1 && \
+            success=`expr $success + 1`
+    grep "status: SERVFAIL" dig.out.ns3.$try > /dev/null 2>&1 && \
+            fail=`expr $fail + 1`
+    stat 50 || ret=1
+    [ $ret -eq 1 ] && break
+    $RNDCCMD recursing 2>&1 | sed 's/^/ns3 /' | cat_i
+    sleep 1
+done
+echo_i "$success successful valid queries, $fail SERVFAIL"
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "checking drop statistics"
+rm -f ns3/named.stats
+$RNDCCMD stats
+for try in 1 2 3 4 5; do
+    [ -f ns3/named.stats ] && break
+    sleep 1
+done
+zspill=`grep 'spilled due to zone' ns3/named.stats | sed 's/\([0-9][0-9]*\) spilled.*/\1/'`
+[ -z "$zspill" ] && zspill=0
+drops=`grep 'queries dropped' ns3/named.stats | sed 's/\([0-9][0-9]*\) queries.*/\1/'`
+[ -z "$drops" ] && drops=0
+[ "$drops" -ge "$zspill" ] || ret=1
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+copy_setports ns3/named3.conf.in ns3/named.conf
+$RNDCCMD reconfig 2>&1 | sed 's/^/ns3 /' | cat_i
+
+echo_i "checking lame server clients are dropped at the soft limit"
+ret=0
+fail=0
+exceeded=0
+success=0
+touch ans4/norespond
+for try in 1 2 3 4 5; do
+    burst b $try 400
+    $DIG @10.53.0.3 -p ${PORT}  a ${try}.example > dig.out.ns3.$try
+    stat 360 || exceeded=`expr $exceeded + 1`
+    grep "status: NOERROR" dig.out.ns3.$try > /dev/null 2>&1 && \
+            success=`expr $success + 1`
+    grep "status: SERVFAIL" dig.out.ns3.$try > /dev/null 2>&1 && \
+            fail=`expr $fail + 1`
+    sleep 1
+done
+echo_i "$success successful valid queries (expected 5)"
+[ "$success" -eq 5 ] || { echo_i "failed"; ret=1; }
+echo_i "$fail SERVFAIL responses (expected 0)"
+[ "$fail" -eq 0 ] || { echo_i "failed"; ret=1; }
+echo_i "clients count exceeded 360 on $exceeded trials (expected 0)"
+[ "$exceeded" -eq 0 ] || { echo_i "failed"; ret=1; }
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1