Merging upstream version 8.2404.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 87 insertions, 3 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 70b0a60..4534776 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -912,16 +912,19 @@ if ENABLE_OMHTTP
 TESTS +=  \
 	omhttp-auth.sh \
 	omhttp-basic.sh \
+	omhttp-basic-ignorecodes.sh \
 	omhttp-batch-fail-with-400.sh \
 	omhttp-batch-jsonarray-compress.sh \
 	omhttp-batch-jsonarray-retry.sh \
 	omhttp-batch-jsonarray.sh \
 	omhttp-batch-kafkarest-retry.sh \
 	omhttp-batch-kafkarest.sh \
+	omhttp-batch-retry-metadata.sh \
 	omhttp-batch-lokirest-retry.sh \
 	omhttp-batch-lokirest.sh \
 	omhttp-batch-newline.sh \
 	omhttp-retry.sh \
+	omhttp-retry-timeout.sh \
 	omhttp-httpheaderkey.sh \
 	omhttp-multiplehttpheaders.sh \
 	omhttp-dynrestpath.sh \
@@ -930,12 +933,15 @@ if HAVE_VALGRIND
 TESTS +=  \
 	omhttp-auth-vg.sh \
 	omhttp-basic-vg.sh \
+	omhttp-basic-ignorecodes-vg.sh \
 	omhttp-batch-jsonarray-compress-vg.sh \
 	omhttp-batch-jsonarray-retry-vg.sh \
 	omhttp-batch-jsonarray-vg.sh \
 	omhttp-batch-kafkarest-retry-vg.sh \
+	omhttp-batch-retry-metadata-vg.sh \
 	omhttp-batch-lokirest-retry-vg.sh \
 	omhttp-retry-vg.sh \
+	omhttp-retry-timeout-vg.sh \
 	omhttp-batch-lokirest-vg.sh
 endif
 endif
@@ -2558,7 +2564,7 @@ EXTRA_DIST= \
 	sndrcv_gzip.sh \
 	imdtls-basic.sh \
 	imdtls-basic-tlscommands.sh \
-	imdtls-basic-timeout \
+	imdtls-basic-timeout.sh \
 	imdtls-error-cert.sh \
 	imdtls-sessionbreak.sh \
 	imdtls-basic-vg.sh \
@@ -2632,6 +2638,7 @@ EXTRA_DIST= \
 	omhttp-batch-lokirest-retry-vg.sh \
 	omhttp-retry-vg.sh \
 	omhttp_server.py \
+	omhttp-validate-response.py \
 	omprog-defaults.sh \
 	omprog-defaults-vg.sh \
 	omprog-output-capture.sh \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 485b8c5..aff58a9 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -836,16 +836,19 @@ host_triplet = @host@
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_46 = \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-auth.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-basic.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-basic-ignorecodes.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-fail-with-400.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-jsonarray-compress.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-jsonarray-retry.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-jsonarray.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-kafkarest-retry.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-kafkarest.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-retry-metadata.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-lokirest-retry.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-lokirest.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-batch-newline.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-retry.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-retry-timeout.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-httpheaderkey.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-multiplehttpheaders.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@	omhttp-dynrestpath.sh \
@@ -854,12 +857,15 @@ host_triplet = @host@
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@am__append_47 = \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-auth-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-basic-vg.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-basic-ignorecodes-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-jsonarray-compress-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-jsonarray-retry-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-jsonarray-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-kafkarest-retry-vg.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-retry-metadata-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-lokirest-retry-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-retry-vg.sh \
+@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-retry-timeout-vg.sh \
 @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@	omhttp-batch-lokirest-vg.sh
 
 @ENABLE_IMKAFKA_TRUE@@ENABLE_KAFKA_TESTS_TRUE@@ENABLE_OMKAFKA_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_48 = \
@@ -3022,7 +3028,7 @@ EXTRA_DIST = \
 	sndrcv_gzip.sh \
 	imdtls-basic.sh \
 	imdtls-basic-tlscommands.sh \
-	imdtls-basic-timeout \
+	imdtls-basic-timeout.sh \
 	imdtls-error-cert.sh \
 	imdtls-sessionbreak.sh \
 	imdtls-basic-vg.sh \
@@ -3096,6 +3102,7 @@ EXTRA_DIST = \
 	omhttp-batch-lokirest-retry-vg.sh \
 	omhttp-retry-vg.sh \
 	omhttp_server.py \
+	omhttp-validate-response.py \
 	omprog-defaults.sh \
 	omprog-defaults-vg.sh \
 	omprog-output-capture.sh \
diff --git a/tests/diag.sh b/tests/diag.sh
index c35edb0..73b93e8 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -2476,6 +2476,20 @@ omhttp_get_data() {
         > ${RSYSLOG_OUT_LOG}
 }
 
+omhttp_validate_metadata_response() {
+	echo "starting to validate omhttp response metadata."
+    omhttp_response_validate_py=$srcdir/omhttp-validate-response.py
+    if [ ! -f $omhttp_response_validate_py ]; then
+        echo "Cannot find ${omhttp_response_validate_py} for omhttp test"
+        error_exit 1
+    fi
+
+	$PYTHON ${omhttp_response_validate_py} --error ${RSYSLOG_DYNNAME}/omhttp.error.log --response ${RSYSLOG_DYNNAME}/omhttp.response.log 2>&1
+	if [ $? -ne 0 ] ; then
+		printf 'omhttp_validate_metadata_response failed \n'
+		error_exit 1 
+	fi
+}
 
 # prepare MySQL for next test
 # each test receives its own database so that we also can run in parallel
diff --git a/tests/imdtls-basic-timeout b/tests/imdtls-basic-timeout.sh
index 9b4c46a..9b4c46a 100755
--- a/tests/imdtls-basic-timeout
+++ b/tests/imdtls-basic-timeout.sh
diff --git a/tests/omhttp-validate-response.py b/tests/omhttp-validate-response.py
new file mode 100644
index 0000000..ef9f09e
--- /dev/null
+++ b/tests/omhttp-validate-response.py
@@ -0,0 +1,34 @@
+import json
+import argparse
+from collections import defaultdict
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser(description='Archive and delete core app log files')
+	parser.add_argument('--error', action='store', type=str, help='error')
+	parser.add_argument('--response', action='store', type=str, help='response')
+	args = parser.parse_args()
+
+	messages = defaultdict(dict)
+	with open(args.error, "r") as error_f, open(args.response, "r") as response_f:
+		for line in error_f:
+			json_obj = json.loads(line)
+			# postdata contains a json string of records array
+			records = json.loads(json_obj['request']['postdata'])
+			if records:
+				for i, val in enumerate(records['records']):
+					messages[val['value']['msgnum']]['response'] = json_obj['response']
+					messages[val['value']['msgnum']]['index'] = i
+		#print (len(messages), "messages:", messages)
+
+		# validate with responses
+		for line in response_f:
+			json_obj = json.loads(line)
+			msgnum = json_obj['message']['msgnum']
+			code = json_obj['response']['code']
+			body = json_obj['response']['body']
+			batch_index = json_obj['response']['batch_index']
+			#print('msgnum:', msgnum, 'code:', code, 'body:', body, 'batch_index:', batch_index)
+			assert(msgnum in messages)
+			assert(messages[msgnum]['response']['status'] == code)
+			assert(messages[msgnum]['response']['message'] == body)
+			assert(messages[msgnum]['index'] == batch_index)
diff --git a/tests/omhttp_server.py b/tests/omhttp_server.py
index 61e0e63..22c7184 100644
--- a/tests/omhttp_server.py
+++ b/tests/omhttp_server.py
@@ -4,6 +4,8 @@ import json
 import os
 import zlib
 import base64
+import random
+import time
 
 try:
     from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer # Python 2
@@ -57,13 +59,27 @@ class MyHandler(BaseHTTPRequestHandler):
                 return
 
         if metadata['fail_with_400_after'] != -1 and metadata['posts'] > metadata['fail_with_400_after']:
+            if metadata['fail_with_delay_secs']:
+                print("sleeping for: {0}".format(metadata['fail_with_delay_secs']))
+                time.sleep(metadata['fail_with_delay_secs'])
             self.send_response(400)
             self.end_headers()
             self.wfile.write(b'BAD REQUEST')
             return
 
+        if metadata['fail_with_401_or_403_after'] != -1 and metadata['posts'] > metadata['fail_with_401_or_403_after']:
+            status = random.choice([401, 403])
+            self.send_response(status)
+            self.end_headers()
+            self.wfile.write(b'BAD REQUEST')
+            return
+
         if metadata['posts'] > 1 and metadata['fail_every'] != -1 and metadata['posts'] % metadata['fail_every'] == 0:
-            self.send_response(500)
+            if metadata['fail_with_delay_secs']:
+                print("sleeping for: {0}".format(metadata['fail_with_delay_secs']))
+                time.sleep(metadata['fail_with_delay_secs'])
+            code = metadata['fail_with'] if metadata['fail_with'] else 500
+            self.send_response(code)
             self.end_headers()
             self.wfile.write(b'INTERNAL ERROR')
             return
@@ -114,13 +130,19 @@ if __name__ == '__main__':
     parser.add_argument('-i', '--interface', action='store', type=str, default='localhost', help='port')
     parser.add_argument('--fail-after', action='store', type=int, default=0, help='start failing after n posts')
     parser.add_argument('--fail-every', action='store', type=int, default=-1, help='fail every n posts')
+    parser.add_argument('--fail-with', action='store', type=int, default=500, help='on failure, fail with this code')
     parser.add_argument('--fail-with-400-after', action='store', type=int, default=-1, help='fail with 400 after n posts')
+    parser.add_argument('--fail-with-401-or-403-after', action='store', type=int, default=-1, help='fail with 401 or 403 after n posts')
+    parser.add_argument('--fail-with-delay-secs', action='store', type=int, default=0, help='fail with n secs of delay')
     parser.add_argument('--decompress', action='store_true', default=False, help='decompress posted data')
     parser.add_argument('--userpwd', action='store', default='', help='only accept this user:password combination')
     args = parser.parse_args()
     metadata['fail_after'] = args.fail_after
     metadata['fail_every'] = args.fail_every
+    metadata['fail_with'] = args.fail_with
     metadata['fail_with_400_after'] = args.fail_with_400_after
+    metadata['fail_with_401_or_403_after'] = args.fail_with_401_or_403_after
+    metadata['fail_with_delay_secs'] = args.fail_with_delay_secs
     metadata['decompress'] = args.decompress
     metadata['userpwd'] = args.userpwd
     server = HTTPServer((args.interface, args.port), MyHandler)