From 3f7feced8779dc78d903d3463f176042598ec24c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 15 Apr 2024 18:28:51 +0200 Subject: Merging upstream version 8.2404.0. Signed-off-by: Daniel Baumann --- .tarball-version | 2 +- ChangeLog | 41 ++++++ Makefile.in | 3 +- configure | 22 +-- configure.ac | 4 +- contrib/omhttp/omhttp.c | 286 +++++++++++++++++++++++++++++++++++++- plugins/imdtls/Makefile.am | 4 +- plugins/imdtls/Makefile.in | 4 +- plugins/imdtls/imdtls.c | 32 ++--- plugins/mmdblookup/mmdblookup.c | 5 + plugins/omdtls/Makefile.am | 4 +- plugins/omdtls/Makefile.in | 4 +- plugins/omdtls/omdtls.c | 20 +-- runtime/net_ossl.c | 29 +++- runtime/net_ossl.h | 39 ++---- runtime/nsd_ossl.c | 25 ++-- tests/Makefile.am | 9 +- tests/Makefile.in | 9 +- tests/diag.sh | 14 ++ tests/imdtls-basic-timeout | 45 ------ tests/imdtls-basic-timeout.sh | 45 ++++++ tests/omhttp-validate-response.py | 34 +++++ tests/omhttp_server.py | 24 +++- tools/rsyslogd.c | 1 - 24 files changed, 557 insertions(+), 148 deletions(-) delete mode 100755 tests/imdtls-basic-timeout create mode 100755 tests/imdtls-basic-timeout.sh create mode 100644 tests/omhttp-validate-response.py diff --git a/.tarball-version b/.tarball-version index 6d5cf31..dddd617 100644 --- a/.tarball-version +++ b/.tarball-version @@ -1 +1 @@ -8.2402.0 +8.2404.0 diff --git a/ChangeLog b/ChangeLog index f4d9a77..fbeea74 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,45 @@ ---------------------------------------------------------------------------------------- +Scheduled Release 8.2404.0 (aka 2024.04) 2024-04-02 +- 2024-04-02: omhttp patches and enhancement + New omhttp plugin configuration parameters added: + * restpathtimeout - configures a timeout value for an omhttp restpath, and retry again + * httpretrycodes (list) - configurable list of HTTP status codes that should be + retried by omhttp plugin. The default behavior is to retry any non 2xx status code. + * httpignorablecodes (list) that can specify HTTP status codes that should be ignored + (as failures). status codes specified as ignorable code will not be retried. + * proxyhost, proxyport - configures a proxy, for which omhttp can use to set up a + HTTP proxy tunnel connection. Also works by reading environment variable if + configured HTTP_PROXY. For more details see libcurl docs on CURLOPT_PROXY + thanks to @erenwh for the contribution! + * retry.addmetadata - when this option is enabled, omhttp will add the + response metadata to: $!omhttp!response. There are 3 response metadata added: + code, body, batch_index. + More statistics omhttp curl statistic counters: + * requests.count + * requests.status.0xx + * requests.status.1xx + * requests.status.2xx + * requests.status.3xx + * requests.status.4xx + * requests.status.5xx + * requests.bytes + * requests.time_ms + No longer discard 3xx, 4xx errors + see also https://github.com/rsyslog/rsyslog/issues/4636 + Thanks to github user n2yen for the patches. +- 2024-04-02: remove CAP_IPC_LOCK capability + Does not seem to be necessary. + Thanks to Michael Biebl for the patch. +- 2024-03-28: TLS bugfix: TLS drivers did not properly load under some conditions + This was a regeression from the DTLS implementation +- 2024-03-28: mmdblookup bugfix: do not suspend on incompatible IP lookup + Mmdblookup module used to suspend after erroring on an IPv6 IP lookup on IPv4-only + DBs. The suspension of the module is now replaced by a simple log message, + allowing it to keep working for future lookups. + Thanks to Théo Bertin (frikilax) for the patch. +- 2024-03-28: tarball fix: dist tarball was missing a textbench file + This made the testbench fail. +---------------------------------------------------------------------------------------- Scheduled Release 8.2402.0 (aka 2024.02) 2024-02-27 - 2024-02-26: add DTLS support This version comes with the initial implementation of imdtls and omdtls. diff --git a/Makefile.in b/Makefile.in index 80682e2..3d9a6e9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -419,8 +419,7 @@ DIST_SUBDIRS = compat runtime grammar . plugins/immark \ contrib/imhiredis tests am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in AUTHORS \ COPYING COPYING.LESSER ChangeLog INSTALL NEWS README compile \ - config.guess config.sub depcomp install-sh ltmain.sh missing \ - ylwrap + config.guess config.sub install-sh ltmain.sh missing ylwrap DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) distdir = $(PACKAGE)-$(VERSION) top_distdir = $(distdir) diff --git a/configure b/configure index 88c0490..2a23f55 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for rsyslog 8.2402.0. +# Generated by GNU Autoconf 2.69 for rsyslog 8.2404.0. # # Report bugs to . # @@ -590,8 +590,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='rsyslog' PACKAGE_TARNAME='rsyslog' -PACKAGE_VERSION='8.2402.0' -PACKAGE_STRING='rsyslog 8.2402.0' +PACKAGE_VERSION='8.2404.0' +PACKAGE_STRING='rsyslog 8.2404.0' PACKAGE_BUGREPORT='rsyslog@lists.adiscon.com' PACKAGE_URL='' @@ -1913,7 +1913,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures rsyslog 8.2402.0 to adapt to many kinds of systems. +\`configure' configures rsyslog 8.2404.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1984,7 +1984,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of rsyslog 8.2402.0:";; + short | recursive ) echo "Configuration of rsyslog 8.2404.0:";; esac cat <<\_ACEOF @@ -2383,7 +2383,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -rsyslog configure 8.2402.0 +rsyslog configure 8.2404.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2963,7 +2963,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by rsyslog $as_me 8.2402.0, which was +It was created by rsyslog $as_me 8.2404.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3315,7 +3315,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu $as_echo "#define VERSION_YEAR 24" >>confdefs.h # UPDATE on release -$as_echo "#define VERSION_MONTH 2" >>confdefs.h +$as_echo "#define VERSION_MONTH 4" >>confdefs.h # UPDATE on release am__api_version='1.16' @@ -3833,7 +3833,7 @@ fi # Define the identity of the package. PACKAGE='rsyslog' - VERSION='8.2402.0' + VERSION='8.2404.0' cat >>confdefs.h <<_ACEOF @@ -31741,7 +31741,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by rsyslog $as_me 8.2402.0, which was +This file was extended by rsyslog $as_me 8.2404.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -31807,7 +31807,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -rsyslog config.status 8.2402.0 +rsyslog config.status 8.2404.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 0de2eed..b86b491 100644 --- a/configure.ac +++ b/configure.ac @@ -2,9 +2,9 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2402.0],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2404.0],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 24, [year part of real rsyslog version]) # UPDATE on release -AC_DEFINE(VERSION_MONTH, 2, [month part of real rsyslog version]) # UPDATE on release +AC_DEFINE(VERSION_MONTH, 4, [month part of real rsyslog version]) # UPDATE on release AM_INIT_AUTOMAKE([subdir-objects]) diff --git a/contrib/omhttp/omhttp.c b/contrib/omhttp/omhttp.c index a2a8111..14b98c1 100644 --- a/contrib/omhttp/omhttp.c +++ b/contrib/omhttp/omhttp.c @@ -83,6 +83,7 @@ STATSCOUNTER_DEF(ctrHttpStatusSuccess, mutCtrHttpStatusSuccess); // Number of re STATSCOUNTER_DEF(ctrHttpStatusFail, mutCtrHttpStatusFail); // Number of requests returning 300+ status static prop_t *pInputName = NULL; +static int omhttpInstancesCnt = 0; #define WRKR_DATA_TYPE_ES 0xBADF0001 @@ -111,6 +112,7 @@ typedef struct instanceConf_s { uchar **serverBaseUrls; int numServers; long healthCheckTimeout; + long restPathTimeout; uchar *uid; uchar *pwd; uchar *authBuf; @@ -123,6 +125,8 @@ typedef struct instanceConf_s { int nHttpHeaders; uchar *restPath; uchar *checkPath; + uchar *proxyHost; + int proxyPort; uchar *tplName; uchar *errorFile; sbool batchMode; @@ -142,6 +146,11 @@ typedef struct instanceConf_s { uchar *myPrivKeyFile; sbool reloadOnHup; sbool retryFailures; + sbool retryAddMetadata; + int nhttpRetryCodes; + unsigned int *httpRetryCodes; + int nIgnorableCodes; + unsigned int *ignorableCodes; unsigned int ratelimitInterval; unsigned int ratelimitBurst; /* for retries */ @@ -149,6 +158,18 @@ typedef struct instanceConf_s { uchar *retryRulesetName; ruleset_t *retryRuleset; struct instanceConf_s *next; + + uchar *statsName; + statsobj_t *stats; + STATSCOUNTER_DEF(ctrHttpRequestsCount, mutCtrHttpRequestsCount); // Number of attempted HTTP requests + STATSCOUNTER_DEF(httpRequestsBytes, mutHttpRequestsBytes); + STATSCOUNTER_DEF(httpRequestsTimeMs, muthttphttpRequestsTimeMs); + STATSCOUNTER_DEF(ctrHttpRequestsStatus0xx, mutCtrHttpRequestsStatus0xx); // HTTP requests returning 0xx + STATSCOUNTER_DEF(ctrHttpRequestsStatus1xx, mutCtrHttpRequestsStatus1xx); // HTTP requests returning 1xx + STATSCOUNTER_DEF(ctrHttpRequestsStatus2xx, mutCtrHttpRequestsStatus2xx); // HTTP requests returning 2xx + STATSCOUNTER_DEF(ctrHttpRequestsStatus3xx, mutCtrHttpRequestsStatus3xx); // HTTP requests returning 3xx + STATSCOUNTER_DEF(ctrHttpRequestsStatus4xx, mutCtrHttpRequestsStatus4xx); // HTTP requests returning 4xx + STATSCOUNTER_DEF(ctrHttpRequestsStatus5xx, mutCtrHttpRequestsStatus5xx); // HTTP requests returning 5xx } instanceData; struct modConfData_s { @@ -190,6 +211,7 @@ static struct cnfparamdescr actpdescr[] = { { "server", eCmdHdlrArray, 0 }, { "serverport", eCmdHdlrInt, 0 }, { "healthchecktimeout", eCmdHdlrInt, 0 }, + { "restpathtimeout", eCmdHdlrInt, 0 }, { "httpcontenttype", eCmdHdlrGetWord, 0 }, { "httpheaderkey", eCmdHdlrGetWord, 0 }, { "httpheadervalue", eCmdHdlrString, 0 }, @@ -199,6 +221,8 @@ static struct cnfparamdescr actpdescr[] = { { "restpath", eCmdHdlrGetWord, 0 }, { "checkpath", eCmdHdlrGetWord, 0 }, { "dynrestpath", eCmdHdlrBinary, 0 }, + { "proxyhost", eCmdHdlrString, 0 }, + { "proxyport", eCmdHdlrInt, 0 }, { "batch", eCmdHdlrBinary, 0 }, { "batch.format", eCmdHdlrGetWord, 0 }, { "batch.maxbytes", eCmdHdlrSize, 0 }, @@ -214,10 +238,14 @@ static struct cnfparamdescr actpdescr[] = { { "tls.mycert", eCmdHdlrString, 0 }, { "tls.myprivkey", eCmdHdlrString, 0 }, { "reloadonhup", eCmdHdlrBinary, 0 }, + { "httpretrycodes", eCmdHdlrArray, 0 }, { "retry", eCmdHdlrBinary, 0 }, + { "retry.addmetadata", eCmdHdlrBinary, 0 }, { "retry.ruleset", eCmdHdlrString, 0 }, { "ratelimit.interval", eCmdHdlrInt, 0 }, { "ratelimit.burst", eCmdHdlrInt, 0 }, + { "name", eCmdHdlrGetWord, 0 }, + { "httpignorablecodes", eCmdHdlrArray, 0 }, }; static struct cnfparamblk actpblk = { CNFPARAMBLK_VERSION, @@ -315,16 +343,23 @@ CODESTARTfreeInstance free(pData->headerBuf); free(pData->restPath); free(pData->checkPath); + free(pData->proxyHost); free(pData->tplName); free(pData->errorFile); free(pData->caCertFile); free(pData->myCertFile); free(pData->myPrivKeyFile); + free(pData->httpRetryCodes); free(pData->retryRulesetName); + free(pData->ignorableCodes); if (pData->ratelimiter != NULL) ratelimitDestruct(pData->ratelimiter); if (pData->bFreeBatchFormatName) free(pData->batchFormatName); + if (pData->stats) { + statsobj.Destruct(&pData->stats); + } + free(pData->statsName); ENDfreeInstance BEGINfreeWrkrInstance @@ -355,6 +390,7 @@ CODESTARTdbgPrintInstInfo dbgprintf("\ttemplate='%s'\n", pData->tplName); dbgprintf("\tnumServers=%d\n", pData->numServers); dbgprintf("\thealthCheckTimeout=%lu\n", pData->healthCheckTimeout); + dbgprintf("\trestPathTimeout=%lu\n", pData->restPathTimeout); dbgprintf("\tserverBaseUrls="); for(i = 0 ; i < pData->numServers ; ++i) dbgprintf("%c'%s'", i == 0 ? '[' : ' ', pData->serverBaseUrls[i]); @@ -375,6 +411,8 @@ CODESTARTdbgPrintInstInfo dbgprintf("\trest path='%s'\n", pData->restPath); dbgprintf("\tcheck path='%s'\n", pData->checkPath); dbgprintf("\tdynamic rest path=%d\n", pData->dynRestPath); + dbgprintf("\tproxy host='%s'\n", pData->proxyHost); + dbgprintf("\tproxy port='%d'\n", pData->proxyPort); dbgprintf("\tuse https=%d\n", pData->useHttps); dbgprintf("\tbatch=%d\n", pData->batchMode); dbgprintf("\tbatch.format='%s'\n", pData->batchFormatName); @@ -390,10 +428,20 @@ CODESTARTdbgPrintInstInfo dbgprintf("\ttls.mycert='%s'\n", pData->myCertFile); dbgprintf("\ttls.myprivkey='%s'\n", pData->myPrivKeyFile); dbgprintf("\treloadonhup='%d'\n", pData->reloadOnHup); + for(i = 0; i < pData->nhttpRetryCodes; ++i) + dbgprintf("%c'%d'", i == 0 ? '[' : ' ', pData->httpRetryCodes[i]); + dbgprintf("]\n"); dbgprintf("\tretry='%d'\n", pData->retryFailures); + dbgprintf("\tretry.addmetadata='%d'\n", pData->retryAddMetadata); dbgprintf("\tretry.ruleset='%s'\n", pData->retryRulesetName); dbgprintf("\tratelimit.interval='%u'\n", pData->ratelimitInterval); dbgprintf("\tratelimit.burst='%u'\n", pData->ratelimitBurst); + for(i = 0; i < pData->nIgnorableCodes; ++i) + dbgprintf("%c'%d'", i == 0 ? '[' : ' ', pData->ignorableCodes[i]); + dbgprintf("]\n"); + dbgprintf("\tratelimit.interval='%d'\n", pData->ratelimitInterval); + dbgprintf("\tratelimit.burst='%d'\n", pData->ratelimitBurst); + dbgprintf("\tstatsname='%s'\n", pData->statsName); ENDdbgPrintInstInfo @@ -755,6 +803,37 @@ finalize_it: RETiRet; } +static rsRetVal +msgAddResponseMetadata(smsg_t *const __restrict__ pMsg, wrkrInstanceData_t *const pWrkrData, size_t batch_index) +{ + struct json_object *json = NULL; + DEFiRet; + CHKmalloc(json = json_object_new_object()); + /* + Following metadata is exposed: + $!omhttp!response!code + $!omhttp!response!body + $!omhttp!response!batch_index + */ + json_object_object_add(json, "code", json_object_new_int(pWrkrData->httpStatusCode)); + if (pWrkrData->reply) { + json_object_object_add(json, "body", json_object_new_string(pWrkrData->reply)); + } + json_object_object_add(json, "batch_index", json_object_new_int(batch_index)); + CHKiRet(msgAddJSON(pMsg, (uchar*)"!omhttp!response", json, 0, 0)); + + /* TODO: possible future, an option to automatically parse to json? + would be under: + $!omhttp!response!parsed + */ + +finalize_it: + if (iRet != RS_RET_OK && json) { + json_object_put(json); + } + RETiRet; +} + static rsRetVal queueBatchOnRetryRuleset(wrkrInstanceData_t *const pWrkrData, instanceData *const pData) { @@ -782,6 +861,12 @@ queueBatchOnRetryRuleset(wrkrInstanceData_t *const pWrkrData, instanceData *cons // And place it on the retry ruleset MsgSetRuleset(pMsg, pData->retryRuleset); + + // Add response specific metadata + if (pData->retryAddMetadata) { + CHKiRet(msgAddResponseMetadata(pMsg, pWrkrData, i)); + } + ratelimitAddMsg(pData->ratelimiter, NULL, pMsg); // Count here in case not entire batch succeeds @@ -798,6 +883,7 @@ checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) long statusCode; size_t numMessages; DEFiRet; + CURLcode resCurl = 0; pData = pWrkrData->pData; statusCode = pWrkrData->httpStatusCode; @@ -814,6 +900,7 @@ checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) if (statusCode == 0) { // request failed, suspend or retry STATSCOUNTER_ADD(ctrMessagesFail, mutCtrMessagesFail, numMessages); + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus0xx, pData->mutCtrHttpRequestsStatus0xx); iRet = RS_RET_SUSPENDED; } else if (statusCode >= 500) { // server error, suspend or retry @@ -824,16 +911,81 @@ checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) // redirection or client error, NO suspend nor retry STATSCOUNTER_INC(ctrHttpStatusFail, mutCtrHttpStatusFail); STATSCOUNTER_ADD(ctrMessagesFail, mutCtrMessagesFail, numMessages); - iRet = RS_RET_DATAFAIL; + iRet = RS_RET_SUSPENDED; + + if (statusCode >= 300 && statusCode < 400) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus3xx, pData->mutCtrHttpRequestsStatus3xx); + } else if (statusCode >= 400 && statusCode < 500) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus4xx, pData->mutCtrHttpRequestsStatus4xx); + } else if (statusCode >= 500 && statusCode < 600) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus5xx, pData->mutCtrHttpRequestsStatus5xx); + } } else { // success, normal state // includes 2XX (success like 200-OK) // includes 1XX (informational like 100-Continue) STATSCOUNTER_INC(ctrHttpStatusSuccess, mutCtrHttpStatusSuccess); STATSCOUNTER_ADD(ctrMessagesSuccess, mutCtrMessagesSuccess, numMessages); + + // increment instance counts if enabled + if (statusCode >= 0 && statusCode < 100) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus0xx, pData->mutCtrHttpRequestsStatus0xx); + } else if (statusCode >= 100 && statusCode < 200) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus1xx, pData->mutCtrHttpRequestsStatus1xx); + } else if (statusCode >= 200 && statusCode < 300) { + STATSCOUNTER_INC(pData->ctrHttpRequestsStatus2xx, pData->mutCtrHttpRequestsStatus2xx); + } iRet = RS_RET_OK; } + // get curl stats for instance + { + long req = 0; + double total = 0; + /* record total bytes */ + resCurl = curl_easy_getinfo(pWrkrData->curlPostHandle, CURLINFO_REQUEST_SIZE, &req); + if (!resCurl) { + STATSCOUNTER_ADD(pWrkrData->pData->httpRequestsBytes, + pWrkrData->pData->mutHttpRequestsBytes, + (uint64_t)req); + } + resCurl = curl_easy_getinfo(pWrkrData->curlPostHandle, CURLINFO_TOTAL_TIME, &total); + if(CURLE_OK == resCurl) { + /* this needs to be converted to milliseconds */ + long total_time_ms = (long)(total*1000); + STATSCOUNTER_ADD(pWrkrData->pData->httpRequestsTimeMs, + pWrkrData->pData->mutHttpRequestsTimeMs, + (uint64_t)total_time_ms); + } + } + + /* when retriable codes are configured, always check status codes */ + if (pData->nhttpRetryCodes) { + sbool bMatch = 0; + for (int i = 0; i < pData->nhttpRetryCodes && pData->httpRetryCodes[i] != 0; ++i) { + if (statusCode == (long)pData->httpRetryCodes[i]) { + bMatch = 1; + break; + } + } + if (bMatch) { + /* just force retry */ + iRet = RS_RET_SUSPENDED; + } else { + iRet = RS_RET_OK; + } + } + + // also check if we can mark this as processed + if (iRet != RS_RET_OK && pData->ignorableCodes) { + for (int i = 0; i < pData->nIgnorableCodes && pData->ignorableCodes[i] != 0; ++i) { + if (statusCode == (long)pData->ignorableCodes[i]) { + iRet = RS_RET_OK; + break; + } + } + } + if (iRet != RS_RET_OK) { LogMsg(0, iRet, LOG_ERR, "omhttp: checkResult error http status code: %ld reply: %s", statusCode, pWrkrData->reply != NULL ? pWrkrData->reply : "NULL"); @@ -1135,6 +1287,7 @@ curlPost(wrkrInstanceData_t *pWrkrData, uchar *message, int msglen, uchar **tpls curlCode = curl_easy_perform(curl); DBGPRINTF("omhttp: curlPost curl returned %lld\n", (long long) curlCode); STATSCOUNTER_INC(ctrHttpRequestCount, mutCtrHttpRequestCount); + STATSCOUNTER_INC(pWrkrData->pData->ctrHttpRequestsCount, pWorkerData->pData->mutCtrHttpRequestsCount); if (curlCode != CURLE_OK) { STATSCOUNTER_INC(ctrHttpRequestFail, mutCtrHttpRequestFail); @@ -1637,6 +1790,15 @@ curlSetupCommon(wrkrInstanceData_t *const pWrkrData, CURL *const handle) curl_easy_setopt(handle, CURLOPT_NOSIGNAL, TRUE); curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); curl_easy_setopt(handle, CURLOPT_WRITEDATA, pWrkrData); + if (pWrkrData->pData->proxyHost != NULL) { + curl_easy_setopt(handle, CURLOPT_PROXY, pWrkrData->pData->proxyHost); + } + if (pWrkrData->pData->proxyPort != 0) { + curl_easy_setopt(handle, CURLOPT_PROXYPORT, pWrkrData->pData->proxyPort); + } + if (pWrkrData->pData->restPathTimeout) { + curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, pWrkrData->pData->restPathTimeout); + } if(pWrkrData->pData->allowUnsignedCerts) curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, FALSE); if(pWrkrData->pData->skipVerifyHost) @@ -1749,6 +1911,7 @@ setInstParamDefaults(instanceData *const pData) pData->defaultPort = 443; pData->healthCheckTimeout = 3500; pData->uid = NULL; + pData->restPathTimeout = 0; pData->httpcontenttype = NULL; pData->headerContentTypeBuf = NULL; pData->httpheaderkey = NULL; @@ -1760,6 +1923,8 @@ setInstParamDefaults(instanceData *const pData) pData->restPath = NULL; pData->checkPath = NULL; pData->dynRestPath = 0; + pData->proxyHost = NULL; + pData->proxyPort = 0; pData->batchMode = 0; pData->batchFormatName = (uchar *)"newline"; pData->batchFormat = FMT_NEWLINE; @@ -1778,11 +1943,18 @@ setInstParamDefaults(instanceData *const pData) pData->myPrivKeyFile = NULL; pData->reloadOnHup= 0; pData->retryFailures = 0; + pData->retryAddMetadata = 0; + pData->nhttpRetryCodes = 0; + pData->httpRetryCodes = NULL; pData->ratelimitBurst = 20000; pData->ratelimitInterval = 600; pData->ratelimiter = NULL; pData->retryRulesetName = NULL; pData->retryRuleset = NULL; + pData->nIgnorableCodes = 0; + pData->ignorableCodes = NULL; + // increment number of instances + ++omhttpInstancesCnt; } static rsRetVal @@ -1828,6 +2000,8 @@ CODESTARTnewActInst pData->defaultPort = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "healthchecktimeout")) { pData->healthCheckTimeout = (long) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "restpathtimeout")) { + pData->restPathTimeout = (long) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "uid")) { pData->uid = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "httpcontenttype")) { @@ -1852,6 +2026,10 @@ CODESTARTnewActInst pData->checkPath = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "dynrestpath")) { pData->dynRestPath = pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "proxyhost")) { + pData->proxyHost = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "proxyport")) { + pData->proxyPort = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "batch")) { pData->batchMode = pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "batch.format")) { @@ -1901,8 +2079,8 @@ CODESTARTnewActInst if(fp == NULL) { rs_strerror_r(errno, errStr, sizeof(errStr)); LogError(0, RS_RET_NO_FILE_ACCESS, - "error: 'tls.cacert' file %s couldn't be accessed: %s\n", - pData->caCertFile, errStr); + "error: 'tls.cacert' file %s couldn't be accessed: %s\n", + pData->caCertFile, errStr); } else { fclose(fp); } @@ -1912,8 +2090,8 @@ CODESTARTnewActInst if(fp == NULL) { rs_strerror_r(errno, errStr, sizeof(errStr)); LogError(0, RS_RET_NO_FILE_ACCESS, - "error: 'tls.mycert' file %s couldn't be accessed: %s\n", - pData->myCertFile, errStr); + "error: 'tls.mycert' file %s couldn't be accessed: %s\n", + pData->myCertFile, errStr); } else { fclose(fp); } @@ -1923,21 +2101,59 @@ CODESTARTnewActInst if(fp == NULL) { rs_strerror_r(errno, errStr, sizeof(errStr)); LogError(0, RS_RET_NO_FILE_ACCESS, - "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n", - pData->myPrivKeyFile, errStr); + "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n", + pData->myPrivKeyFile, errStr); } else { fclose(fp); } } else if(!strcmp(actpblk.descr[i].name, "reloadonhup")) { pData->reloadOnHup= pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "httpretrycodes")) { + pData->nhttpRetryCodes = pvals[i].val.d.ar->nmemb; + // note: use zero as sentinel value + CHKmalloc(pData->httpRetryCodes = calloc(pvals[i].val.d.ar->nmemb, sizeof(unsigned int) )); + int count = 0; + for(int j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { + int bSuccess = 0; + long long n = es_str2num(pvals[i].val.d.ar->arr[j], &bSuccess); + if (!bSuccess) { + char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + LogError(0, RS_RET_NO_FILE_ACCESS, + "error: 'httpRetryCode' '%s' is not a number - ignored\n", cstr); + free(cstr); + } else { + pData->httpRetryCodes[count++] = n; + } + } } else if(!strcmp(actpblk.descr[i].name, "retry")) { pData->retryFailures = pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "retry.ruleset")) { pData->retryRulesetName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "retry.addmetadata")) { + pData->retryAddMetadata = pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "ratelimit.burst")) { pData->ratelimitBurst = (unsigned int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "ratelimit.interval")) { pData->ratelimitInterval = (unsigned int) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "name")) { + pData->statsName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "httpignorablecodes")) { + pData->nIgnorableCodes = pvals[i].val.d.ar->nmemb; + // note: use zero as sentinel value + CHKmalloc(pData->ignorableCodes = calloc(pvals[i].val.d.ar->nmemb, sizeof(unsigned int))); + int count = 0; + for(int j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { + int bSuccess = 0; + long long n = es_str2num(pvals[i].val.d.ar->arr[j], &bSuccess); + if (!bSuccess) { + char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + LogError(0, RS_RET_NO_FILE_ACCESS, + "error: 'httpIgnorableCodes' '%s' is not a number - ignored\n", cstr); + free(cstr); + } else { + pData->ignorableCodes[count++] = n; + } + } } else { LogError(0, RS_RET_INTERNAL_ERROR, "omhttp: program error, " "non-handled param '%s'", actpblk.descr[i].name); @@ -1963,6 +2179,14 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } + if (pData->proxyHost == NULL) { + if (getenv("http_proxy") != NULL) { + pData->proxyHost = ustrdup(getenv("http_proxy")); + } else if (getenv("HTTP_PROXY") != NULL) { + pData->proxyHost = ustrdup(getenv("HTTP_PROXY")); + } + } + if (pData->uid != NULL) CHKiRet(computeAuthHeader((char*) pData->uid, (char*) pData->pwd, &pData->authBuf)); if (pData->httpcontenttype != NULL) @@ -2039,6 +2263,54 @@ CODESTARTnewActInst ratelimitSetNoTimeCache(pData->ratelimiter); } + if(!pData->statsName) { + uchar pszAName[64]; + snprintf((char*) pszAName, sizeof(pszAName), "omhttp-%d", omhttpInstancesCnt); + pData->statsName = ustrdup(pszAName); + } + // instantiate the stats object and add the counters + CHKiRet(statsobj.Construct(&pData->stats)); + CHKiRet(statsobj.SetName(pData->stats, (uchar *)pData->statsName)); + CHKiRet(statsobj.SetOrigin(pData->stats, (uchar *)"omhttp")); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsCount, pData->mutCtrHttpRequestsCount); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.count", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsCount)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus0xx, pData->mutCtrHttpRequestsStatus0xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.0xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus0xx)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus1xx, pData->mutCtrHttpRequestsStatus1xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.1xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus1xx)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus2xx, pData->mutCtrHttpRequestsStatus2xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.2xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus2xx)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus3xx, pData->mutCtrHttpRequestsStatus3xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.3xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus3xx)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus4xx, pData->mutCtrHttpRequestsStatus4xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.4xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus4xx)); + + STATSCOUNTER_INIT(pData->ctrHttpRequestsStatus5xx, pData->mutCtrHttpRequestsStatus5xx); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.status.5xx", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->ctrHttpRequestsStatus5xx)); + + STATSCOUNTER_INIT(pData->httpRequestsBytes, pData->mutHttpRequestsBytes); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.bytes", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->httpRequestsBytes)); + + STATSCOUNTER_INIT(pData->httpRequestsTimeMs, pData->mutHttpRequestsTimeMs); + CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"requests.time_ms", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &pData->httpRequestsTimeMs)); + + CHKiRet(statsobj.ConstructFinalize(pData->stats)); + /* node created, let's add to list of instance configs for the module */ if(loadModConf->tail == NULL) { loadModConf->tail = loadModConf->root = pData; diff --git a/plugins/imdtls/Makefile.am b/plugins/imdtls/Makefile.am index bf544b3..3253444 100644 --- a/plugins/imdtls/Makefile.am +++ b/plugins/imdtls/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imdtls.la -imdtls_la_DEPENDENCIES = ../../runtime/lmnsd_ossl.la +imdtls_la_DEPENDENCIES = imdtls_la_SOURCES = imdtls.c imdtls_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(OPENSSL_CFLAGS) imdtls_la_LDFLAGS = -module -avoid-version -imdtls_la_LIBADD = $(OPENSSL_LIBS) ../../runtime/lmnsd_ossl.la +imdtls_la_LIBADD = $(OPENSSL_LIBS) diff --git a/plugins/imdtls/Makefile.in b/plugins/imdtls/Makefile.in index 03043f4..d9a5d48 100644 --- a/plugins/imdtls/Makefile.in +++ b/plugins/imdtls/Makefile.in @@ -451,11 +451,11 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = imdtls.la -imdtls_la_DEPENDENCIES = ../../runtime/lmnsd_ossl.la +imdtls_la_DEPENDENCIES = imdtls_la_SOURCES = imdtls.c imdtls_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(OPENSSL_CFLAGS) imdtls_la_LDFLAGS = -module -avoid-version -imdtls_la_LIBADD = $(OPENSSL_LIBS) ../../runtime/lmnsd_ossl.la +imdtls_la_LIBADD = $(OPENSSL_LIBS) all: all-am .SUFFIXES: diff --git a/plugins/imdtls/imdtls.c b/plugins/imdtls/imdtls.c index 6501d9c..3751bbe 100644 --- a/plugins/imdtls/imdtls.c +++ b/plugins/imdtls/imdtls.c @@ -314,24 +314,24 @@ imdtls_verify_callback(int status, SSL* ssl) switch(inst->pNetOssl->authMode) { case OSSL_AUTH_CERTNAME: /* if we check the name, we must ensure the cert is valid */ - certpeer = net_ossl_getpeercert(inst->pNetOssl, ssl, NULL); + certpeer = net_ossl.osslGetpeercert(inst->pNetOssl, ssl, NULL); dbgprintf("imdtls_verify_callback: Check peer certname[%p]=%s\n", (void *)ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(inst->pNetOssl, ssl, NULL)); - CHKiRet(net_ossl_chkpeername(inst->pNetOssl, certpeer, NULL)); + CHKiRet(net_ossl.osslChkpeercertvalidity(inst->pNetOssl, ssl, NULL)); + CHKiRet(net_ossl.osslChkpeername(inst->pNetOssl, certpeer, NULL)); break; case OSSL_AUTH_CERTFINGERPRINT: - certpeer = net_ossl_getpeercert(inst->pNetOssl, ssl, NULL); + certpeer = net_ossl.osslGetpeercert(inst->pNetOssl, ssl, NULL); dbgprintf("imdtls_verify_callback: Check peer fingerprint[%p]=%s\n", (void *)ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(inst->pNetOssl, ssl, NULL)); - CHKiRet(net_ossl_peerfingerprint(inst->pNetOssl, certpeer, NULL)); + CHKiRet(net_ossl.osslChkpeercertvalidity(inst->pNetOssl, ssl, NULL)); + CHKiRet(net_ossl.osslPeerfingerprint(inst->pNetOssl, certpeer, NULL)); break; case OSSL_AUTH_CERTVALID: - certpeer = net_ossl_getpeercert(inst->pNetOssl, ssl, NULL); + certpeer = net_ossl.osslGetpeercert(inst->pNetOssl, ssl, NULL); dbgprintf("imdtls_verify_callback: Check peer valid[%p]=%s\n", (void *)ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(inst->pNetOssl, ssl, NULL)); + CHKiRet(net_ossl.osslChkpeercertvalidity(inst->pNetOssl, ssl, NULL)); break; case OSSL_AUTH_CERTANON: dbgprintf("imdtls_verify_callback: ANON[%p]\n", (void *)ssl); @@ -401,7 +401,7 @@ addListner(modConfData_t __attribute__((unused)) *modConf, instanceConf_t *inst) CHKiRet(net_ossl.osslCtxInitCookie(inst->pNetOssl)); # endif // Run openssl config commands in Context - CHKiRet(net_ossl_apply_tlscgfcmd(inst->pNetOssl, inst->tlscfgcmd)); + CHKiRet(net_ossl.osslApplyTlscgfcmd(inst->pNetOssl, inst->tlscfgcmd)); // Init Socket CHKiRet(DTLSCreateSocket(inst)); @@ -499,13 +499,13 @@ DTLSAcceptSession(instanceConf_t *inst, int idx) { } else if(err == SSL_ERROR_SYSCALL) { DBGPRINTF("imdtls: SSL_accept failed SSL_ERROR_SYSCALL idx (%d), removing client.\n", idx); - net_ossl_lastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, + net_ossl.osslLastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, "DTLSHandleSessions", "SSL_accept"); DTLScleanupSession(inst, idx); } else { // An actual error occurred DBGPRINTF("imdtls: SSL_accept failed (%d) idx (%d), removing client.\n", err, idx); - net_ossl_lastOpenSSLErrorMsg(NULL, err, ssl, LOG_ERR, + net_ossl.osslLastOpenSSLErrorMsg(NULL, err, ssl, LOG_ERR, "DTLSHandleSessions", "SSL_accept"); DTLScleanupSession(inst, idx); } @@ -570,7 +570,7 @@ DTLSReadClient(instanceConf_t *inst, int idx, short revents) { break; } else if (err == SSL_ERROR_SYSCALL) { DBGPRINTF("imdtls: SSL_ERROR_SYSCALL on index %d ERRNO %d\n", idx, errno); - net_ossl_lastOpenSSLErrorMsg(NULL, err, ssl, LOG_ERR, + net_ossl.osslLastOpenSSLErrorMsg(NULL, err, ssl, LOG_ERR, "DTLSReadClient", "SSL_read"); DTLScleanupSession(inst, idx); break; @@ -655,7 +655,7 @@ DTLSHandleSessions(instanceConf_t *inst) { if (inst->pNetOssl->authMode != OSSL_AUTH_CERTANON) { dbgprintf("imdtls: enable certificate checking (Mode=%d, VerifyDepth=%d)\n", inst->pNetOssl->authMode, inst->CertVerifyDepth); - net_ossl_set_ssl_verify_callback(ssl, + net_ossl.osslSetSslVerifyCallback(ssl, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT); if (inst->CertVerifyDepth != 0) { SSL_set_verify_depth(ssl, inst->CertVerifyDepth); @@ -668,7 +668,7 @@ DTLSHandleSessions(instanceConf_t *inst) { SSL_set_ex_data(ssl, 2, inst); /* Used in imdtls */ // Debug Callback for conn sbio! - net_ossl_set_bio_callback(sbio); + net_ossl.osslSetBioCallback(sbio); // Connect the new Client BIO_ADDR *client_addr = BIO_ADDR_new(); @@ -711,7 +711,7 @@ DTLSHandleSessions(instanceConf_t *inst) { if (ret == 0) { err = SSL_get_error(ssl, ret); DBGPRINTF("imdtls: DTLSHandleSessions BIO_connect ERROR %d\n", err); - net_ossl_lastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, + net_ossl.osslLastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, "DTLSHandleSessions", "BIO_connect"); LogMsg(0, RS_RET_NO_ERRCODE, LOG_WARNING, "imdtls: BIO_connect failed for DTLS client"); @@ -744,7 +744,7 @@ DTLSHandleSessions(instanceConf_t *inst) { } else { DBGPRINTF("imdtls: DTLSv1_listen RET %d (ERR %d / ERRNO %d), abort\n", ret, err, errno); - net_ossl_lastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, + net_ossl.osslLastOpenSSLErrorMsg(NULL, err, ssl, LOG_WARNING, "DTLSHandleSessions", "DTLSv1_listen"); LogMsg(0, RS_RET_NO_ERRCODE, LOG_WARNING, "imdtls: DTLSv1_listen failed for DTLS client"); diff --git a/plugins/mmdblookup/mmdblookup.c b/plugins/mmdblookup/mmdblookup.c index f9f3c73..d6a26f7 100644 --- a/plugins/mmdblookup/mmdblookup.c +++ b/plugins/mmdblookup/mmdblookup.c @@ -412,6 +412,11 @@ CODESTARTdoAction dbgprintf("Error from call to getaddrinfo for %s - %s\n", pszValue, gai_strerror(gai_err)); ABORT_FINALIZE(RS_RET_OK); } + if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_err) { + LogMsg(0, NO_ERRCODE, LOG_INFO, "mmdblookup: Tried to search for an IPv6 address in an IPv4-only DB" + ", ignoring"); + ABORT_FINALIZE(RS_RET_OK); + } if (MMDB_SUCCESS != mmdb_err) { dbgprintf("Got an error from the maxminddb library: %s\n", MMDB_strerror(mmdb_err)); close_mmdb(&pWrkrData->mmdb); diff --git a/plugins/omdtls/Makefile.am b/plugins/omdtls/Makefile.am index 8451028..a877419 100644 --- a/plugins/omdtls/Makefile.am +++ b/plugins/omdtls/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omdtls.la -omdtls_la_DEPENDENCIES = ../../runtime/lmnsd_ossl.la +omdtls_la_DEPENDENCIES = omdtls_la_SOURCES = omdtls.c omdtls_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(OPENSSL_CFLAGS) omdtls_la_LDFLAGS = -module -avoid-version -omdtls_la_LIBADD = $(OPENSSL_LIBS) ../../runtime/lmnsd_ossl.la +omdtls_la_LIBADD = $(OPENSSL_LIBS) diff --git a/plugins/omdtls/Makefile.in b/plugins/omdtls/Makefile.in index 6978ece..d06d59c 100644 --- a/plugins/omdtls/Makefile.in +++ b/plugins/omdtls/Makefile.in @@ -451,11 +451,11 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = omdtls.la -omdtls_la_DEPENDENCIES = ../../runtime/lmnsd_ossl.la +omdtls_la_DEPENDENCIES = omdtls_la_SOURCES = omdtls.c omdtls_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(OPENSSL_CFLAGS) omdtls_la_LDFLAGS = -module -avoid-version -omdtls_la_LIBADD = $(OPENSSL_LIBS) ../../runtime/lmnsd_ossl.la +omdtls_la_LIBADD = $(OPENSSL_LIBS) all: all-am .SUFFIXES: diff --git a/plugins/omdtls/omdtls.c b/plugins/omdtls/omdtls.c index c5ba167..dd4c55f 100644 --- a/plugins/omdtls/omdtls.c +++ b/plugins/omdtls/omdtls.c @@ -270,7 +270,7 @@ CODESTARTactivateCnfPrePrivDrop for(inst = runModConf->root ; inst != NULL ; inst = inst->next) { CHKiRet(net_ossl.osslCtxInit(inst->pNetOssl, DTLS_method())); // Run openssl config commands in Context - CHKiRet(net_ossl_apply_tlscgfcmd(inst->pNetOssl, inst->tlscfgcmd)); + CHKiRet(net_ossl.osslApplyTlscgfcmd(inst->pNetOssl, inst->tlscfgcmd)); } finalize_it: ENDactivateCnfPrePrivDrop @@ -598,13 +598,13 @@ dtls_send(wrkrInstanceData_t *pWrkrData, const actWrkrIParams_t *__restrict__ co if (sslerr == SSL_ERROR_SYSCALL) { dbgprintf("dtls_send[%p]: SSL_write failed with SSL_ERROR_SYSCALL(%s)" " - Aborting Connection.\n", pWrkrData, strerror(errno)); - net_ossl_lastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_WARNING, + net_ossl.osslLastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_WARNING, "omdtls", "SSL_write"); ABORT_FINALIZE(RS_RET_ERR); } else { dbgprintf("dtls_send[%p]: SSL_write failed with ERROR [%d]: %s" " - Aborting Connection.\n", pWrkrData, sslerr, ERR_error_string(sslerr, NULL)); - net_ossl_lastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_WARNING, + net_ossl.osslLastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_WARNING, "omdtls", "SSL_write"); ABORT_FINALIZE(RS_RET_ERR); } @@ -639,7 +639,8 @@ dtls_connect(wrkrInstanceData_t *pWrkrData) { pWrkrData->sslClient = SSL_new(pData->pNetOssl->ctx); if(!pWrkrData->sslClient) { dbgprintf("dtls_connect[%p]: SSL_new failed failed\n", pWrkrData); - net_ossl_lastOpenSSLErrorMsg(pData->target, 0, pWrkrData->sslClient, LOG_WARNING, "omdtls", "SSL_new"); + net_ossl.osslLastOpenSSLErrorMsg(pData->target, 0, pWrkrData->sslClient, + LOG_WARNING, "omdtls", "SSL_new"); ABORT_FINALIZE(RS_RET_ERR); } @@ -648,19 +649,20 @@ dtls_connect(wrkrInstanceData_t *pWrkrData) { dbgprintf("dtls_connect[%p]: enable certificate checking (Mode=%d, VerifyDepth=%d)\n", pWrkrData, pData->pNetOssl->authMode, pData->CertVerifyDepth); /* Enable certificate valid checking */ - net_ossl_set_ssl_verify_callback(pWrkrData->sslClient, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT); + net_ossl.osslSetSslVerifyCallback(pWrkrData->sslClient, + SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT); if (pData->CertVerifyDepth != 0) { SSL_set_verify_depth(pWrkrData->sslClient, pData->CertVerifyDepth); } } else { dbgprintf("dtls_connect[%p]: disable certificate checking\n", pWrkrData); - net_ossl_set_ssl_verify_callback(pWrkrData->sslClient, SSL_VERIFY_NONE); + net_ossl.osslSetSslVerifyCallback(pWrkrData->sslClient, SSL_VERIFY_NONE); } /* Create BIO from socket array! */ bio_client = BIO_new_dgram(pWrkrData->sockout, BIO_NOCLOSE); if (!bio_client) { - net_ossl_lastOpenSSLErrorMsg(pData->target, 0, pWrkrData->sslClient, LOG_INFO, + net_ossl.osslLastOpenSSLErrorMsg(pData->target, 0, pWrkrData->sslClient, LOG_INFO, "dtls_connect", "BIO_new_dgram"); ABORT_FINALIZE(RS_RET_ERR); } @@ -668,13 +670,13 @@ dtls_connect(wrkrInstanceData_t *pWrkrData) { SSL_set_bio(pWrkrData->sslClient, bio_client, bio_client); /* Set debug Callback for conn BIO as well! */ - net_ossl_set_bio_callback(bio_client); + net_ossl.osslSetBioCallback(bio_client); dbgprintf("dtls_connect[%p]: Starting DTLS session ...\n", pWrkrData); /* Perform handshake */ iErr = SSL_connect(pWrkrData->sslClient); if (iErr <= 0) { - net_ossl_lastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_INFO, + net_ossl.osslLastOpenSSLErrorMsg(pData->target, iErr, pWrkrData->sslClient, LOG_INFO, "dtls_connect", "SSL_connect"); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/runtime/net_ossl.c b/runtime/net_ossl.c index 60e3fa2..7008731 100644 --- a/runtime/net_ossl.c +++ b/runtime/net_ossl.c @@ -52,6 +52,20 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(net) DEFobjCurrIf(nsd_ptcp) +/* Prototypes for openssl helper functions */ +void net_ossl_lastOpenSSLErrorMsg + (uchar *fromHost, int ret, SSL *ssl, int severity, const char* pszCallSource, const char* pszOsslApi); +void net_ossl_set_ssl_verify_callback(SSL *pSsl, int flags); +void net_ossl_set_ctx_verify_callback(SSL_CTX *pCtx, int flags); +void net_ossl_set_bio_callback(BIO *conn); +int net_ossl_verify_callback(int status, X509_STORE_CTX *store); +rsRetVal net_ossl_apply_tlscgfcmd(net_ossl_t *pThis, uchar *tlscfgcmd); +rsRetVal net_ossl_chkpeercertvalidity(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); +X509* net_ossl_getpeercert(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); +rsRetVal net_ossl_peerfingerprint(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); +rsRetVal net_ossl_chkpeername(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); + + /*--------------------------------------MT OpenSSL helpers ------------------------------------------*/ static MUTEX_TYPE *mutex_buf = NULL; static sbool openssl_initialized = 0; // Avoid multiple initialization / deinitialization @@ -1174,9 +1188,18 @@ CODESTARTobjQueryInterface(net_ossl) if(pIf->ifVersion != net_osslCURR_IF_VERSION) {/* check for current version, increment on each change */ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); } - pIf->Construct = (rsRetVal(*)(net_ossl_t**)) net_osslConstruct; - pIf->Destruct = (rsRetVal(*)(net_ossl_t**)) net_osslDestruct; - pIf->osslCtxInit = net_ossl_osslCtxInit; + pIf->Construct = (rsRetVal(*)(net_ossl_t**)) net_osslConstruct; + pIf->Destruct = (rsRetVal(*)(net_ossl_t**)) net_osslDestruct; + pIf->osslCtxInit = net_ossl_osslCtxInit; + pIf->osslChkpeername = net_ossl_chkpeername; + pIf->osslPeerfingerprint = net_ossl_peerfingerprint; + pIf->osslGetpeercert = net_ossl_getpeercert; + pIf->osslChkpeercertvalidity = net_ossl_chkpeercertvalidity; + pIf->osslApplyTlscgfcmd = net_ossl_apply_tlscgfcmd; + pIf->osslSetBioCallback = net_ossl_set_bio_callback; + pIf->osslSetCtxVerifyCallback = net_ossl_set_ctx_verify_callback; + pIf->osslSetSslVerifyCallback = net_ossl_set_ssl_verify_callback; + pIf->osslLastOpenSSLErrorMsg = net_ossl_lastOpenSSLErrorMsg; #if OPENSSL_VERSION_NUMBER >= 0x10100000L pIf->osslCtxInitCookie = net_ossl_ctx_init_cookie; #endif diff --git a/runtime/net_ossl.h b/runtime/net_ossl.h index 6e8a61f..eef69dd 100644 --- a/runtime/net_ossl.h +++ b/runtime/net_ossl.h @@ -83,6 +83,17 @@ BEGINinterface(net_ossl) /* name must also be changed in ENDinterface macro! */ #if OPENSSL_VERSION_NUMBER >= 0x10100000L rsRetVal (*osslCtxInitCookie)(net_ossl_t *pThis); #endif // OPENSSL_VERSION_NUMBER >= 0x10100000L + // OpenSSL Helper function exports + rsRetVal (*osslChkpeername)(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); + rsRetVal (*osslPeerfingerprint)(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); + X509* (*osslGetpeercert)(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); + rsRetVal (*osslChkpeercertvalidity)(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); + rsRetVal (*osslApplyTlscgfcmd)(net_ossl_t *pThis, uchar *tlscfgcmd); + void (*osslSetBioCallback)(BIO *conn); + void (*osslSetCtxVerifyCallback)(SSL_CTX *pCtx, int flags); + void (*osslSetSslVerifyCallback)(SSL *pSsl, int flags); + void (*osslLastOpenSSLErrorMsg)(uchar *fromHost, + const int ret, SSL *ssl, int severity, const char* pszCallSource, const char* pszOsslApi); ENDinterface(net_ossl) #define net_osslCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ @@ -134,34 +145,6 @@ void osslGlblExit(void); /*-----------------------------------------------------------------------------*/ -/* Prototypes for openssl helper functions */ -__attribute__((visibility("default"))) void net_ossl_lastOpenSSLErrorMsg - (uchar *fromHost, const int ret, SSL *ssl, int severity, const char* pszCallSource, const char* pszOsslApi); -__attribute__((visibility("default"))) void net_ossl_set_ssl_verify_callback(SSL *pSsl, int flags); -__attribute__((visibility("default"))) void net_ossl_set_ctx_verify_callback(SSL_CTX *pCtx, int flags); -__attribute__((visibility("default"))) void net_ossl_set_bio_callback(BIO *conn); -__attribute__((visibility("default"))) int net_ossl_verify_callback(int status, X509_STORE_CTX *store); -__attribute__((visibility("default"))) rsRetVal net_ossl_apply_tlscgfcmd(net_ossl_t *pThis, uchar *tlscfgcmd); -__attribute__((visibility("default"))) rsRetVal - net_ossl_chkpeercertvalidity(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); -__attribute__((visibility("default"))) X509* - net_ossl_getpeercert(net_ossl_t *pThis, SSL *ssl, uchar *fromHostIP); -__attribute__((visibility("default"))) rsRetVal - net_ossl_peerfingerprint(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); -__attribute__((visibility("default"))) rsRetVal - net_ossl_chkpeername(net_ossl_t *pThis, X509* certpeer, uchar *fromHostIP); - -/* -#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) -long RSYSLOG_BIO_debug_callback_ex(BIO *bio, int cmd, const char __attribute__((unused)) *argp, - size_t __attribute__((unused)) len, int argi, long __attribute__((unused)) argl, - int ret, size_t __attribute__((unused)) *processed); -#else -long RSYSLOG_BIO_debug_callback(BIO *bio, int cmd, const char __attribute__((unused)) *argp, - int argi, long __attribute__((unused)) argl, long ret); -#endif -*/ - /* prototypes */ PROTOTYPEObj(net_ossl); diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 2d70fb6..095328b 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -80,7 +80,7 @@ void nsd_ossl_lastOpenSSLErrorMsg(nsd_ossl_t const *pThis, const int ret, SSL *s } // Call helper in net_ossl - net_ossl_lastOpenSSLErrorMsg(fromHost, ret, ssl, severity, pszCallSource, pszOsslApi); + net_ossl.osslLastOpenSSLErrorMsg(fromHost, ret, ssl, severity, pszCallSource, pszOsslApi); free(fromHost); errno = errno_store; @@ -278,7 +278,8 @@ osslInitSession(nsd_ossl_t *pThis, osslSslState_t osslType) /* , nsd_ossl_t *pSe dbgprintf("osslInitSession: enable certificate checking (Mode=%d, VerifyDepth=%d)\n", pThis->pNetOssl->authMode, pThis->DrvrVerifyDepth); /* Enable certificate valid checking */ - net_ossl_set_ssl_verify_callback(pThis->pNetOssl->ssl, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT); + net_ossl.osslSetSslVerifyCallback(pThis->pNetOssl->ssl, + SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT); if (pThis->DrvrVerifyDepth != 0) { SSL_set_verify_depth(pThis->pNetOssl->ssl, pThis->DrvrVerifyDepth); } @@ -305,7 +306,7 @@ osslInitSession(nsd_ossl_t *pThis, osslSslState_t osslType) /* , nsd_ossl_t *pSe dbgprintf("osslInitSession: Init conn BIO[%p] done\n", (void *)conn); /* Set debug Callback for conn BIO as well! */ - net_ossl_set_bio_callback(conn); + net_ossl.osslSetBioCallback(conn); /* TODO: still needed? Set to NON blocking ! */ BIO_set_nbio( conn, 1 ); @@ -347,25 +348,25 @@ osslChkPeerAuth(nsd_ossl_t *pThis) switch(pThis->pNetOssl->authMode) { case OSSL_AUTH_CERTNAME: /* if we check the name, we must ensure the cert is valid */ - certpeer = net_ossl_getpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); + certpeer = net_ossl.osslGetpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); dbgprintf("osslChkPeerAuth: Check peer certname[%p]=%s\n", (void *)pThis->pNetOssl->ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); - CHKiRet(net_ossl_chkpeername(pThis->pNetOssl, certpeer, fromHostIP)); + CHKiRet(net_ossl.osslChkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); + CHKiRet(net_ossl.osslChkpeername(pThis->pNetOssl, certpeer, fromHostIP)); break; case OSSL_AUTH_CERTFINGERPRINT: - certpeer = net_ossl_getpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); + certpeer = net_ossl.osslGetpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); dbgprintf("osslChkPeerAuth: Check peer fingerprint[%p]=%s\n", (void *)pThis->pNetOssl->ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); - CHKiRet(net_ossl_peerfingerprint(pThis->pNetOssl, certpeer, fromHostIP)); + CHKiRet(net_ossl.osslChkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); + CHKiRet(net_ossl.osslPeerfingerprint(pThis->pNetOssl, certpeer, fromHostIP)); break; case OSSL_AUTH_CERTVALID: - certpeer = net_ossl_getpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); + certpeer = net_ossl.osslGetpeercert(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP); dbgprintf("osslChkPeerAuth: Check peer valid[%p]=%s\n", (void *)pThis->pNetOssl->ssl, (certpeer != NULL ? "VALID" : "NULL")); - CHKiRet(net_ossl_chkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); + CHKiRet(net_ossl.osslChkpeercertvalidity(pThis->pNetOssl, pThis->pNetOssl->ssl, fromHostIP)); break; case OSSL_AUTH_CERTANON: FINALIZE; @@ -1277,7 +1278,7 @@ applyGnutlsPriorityString(nsd_ossl_t *const pThis) if(pThis->gnutlsPriorityString == NULL || pThis->pNetOssl->ctx == NULL) { FINALIZE; } else { - CHKiRet(net_ossl_apply_tlscgfcmd(pThis->pNetOssl, pThis->gnutlsPriorityString)); + CHKiRet(net_ossl.osslApplyTlscgfcmd(pThis->pNetOssl, pThis->gnutlsPriorityString)); } #endif diff --git a/tests/Makefile.am b/tests/Makefile.am index 70b0a60..4534776 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -912,16 +912,19 @@ if ENABLE_OMHTTP TESTS += \ omhttp-auth.sh \ omhttp-basic.sh \ + omhttp-basic-ignorecodes.sh \ omhttp-batch-fail-with-400.sh \ omhttp-batch-jsonarray-compress.sh \ omhttp-batch-jsonarray-retry.sh \ omhttp-batch-jsonarray.sh \ omhttp-batch-kafkarest-retry.sh \ omhttp-batch-kafkarest.sh \ + omhttp-batch-retry-metadata.sh \ omhttp-batch-lokirest-retry.sh \ omhttp-batch-lokirest.sh \ omhttp-batch-newline.sh \ omhttp-retry.sh \ + omhttp-retry-timeout.sh \ omhttp-httpheaderkey.sh \ omhttp-multiplehttpheaders.sh \ omhttp-dynrestpath.sh \ @@ -930,12 +933,15 @@ if HAVE_VALGRIND TESTS += \ omhttp-auth-vg.sh \ omhttp-basic-vg.sh \ + omhttp-basic-ignorecodes-vg.sh \ omhttp-batch-jsonarray-compress-vg.sh \ omhttp-batch-jsonarray-retry-vg.sh \ omhttp-batch-jsonarray-vg.sh \ omhttp-batch-kafkarest-retry-vg.sh \ + omhttp-batch-retry-metadata-vg.sh \ omhttp-batch-lokirest-retry-vg.sh \ omhttp-retry-vg.sh \ + omhttp-retry-timeout-vg.sh \ omhttp-batch-lokirest-vg.sh endif endif @@ -2558,7 +2564,7 @@ EXTRA_DIST= \ sndrcv_gzip.sh \ imdtls-basic.sh \ imdtls-basic-tlscommands.sh \ - imdtls-basic-timeout \ + imdtls-basic-timeout.sh \ imdtls-error-cert.sh \ imdtls-sessionbreak.sh \ imdtls-basic-vg.sh \ @@ -2632,6 +2638,7 @@ EXTRA_DIST= \ omhttp-batch-lokirest-retry-vg.sh \ omhttp-retry-vg.sh \ omhttp_server.py \ + omhttp-validate-response.py \ omprog-defaults.sh \ omprog-defaults-vg.sh \ omprog-output-capture.sh \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 485b8c5..aff58a9 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -836,16 +836,19 @@ host_triplet = @host@ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_46 = \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-auth.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-basic.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-basic-ignorecodes.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-fail-with-400.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-jsonarray-compress.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-jsonarray-retry.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-jsonarray.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-kafkarest-retry.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-kafkarest.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-retry-metadata.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-lokirest-retry.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-lokirest.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-batch-newline.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-retry.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-retry-timeout.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-httpheaderkey.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-multiplehttpheaders.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@ omhttp-dynrestpath.sh \ @@ -854,12 +857,15 @@ host_triplet = @host@ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@am__append_47 = \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-auth-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-basic-vg.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-basic-ignorecodes-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-jsonarray-compress-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-jsonarray-retry-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-jsonarray-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-kafkarest-retry-vg.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-retry-metadata-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-lokirest-retry-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-retry-vg.sh \ +@ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-retry-timeout-vg.sh \ @ENABLE_OMHTTP_TRUE@@ENABLE_TESTBENCH_TRUE@@HAVE_VALGRIND_TRUE@ omhttp-batch-lokirest-vg.sh @ENABLE_IMKAFKA_TRUE@@ENABLE_KAFKA_TESTS_TRUE@@ENABLE_OMKAFKA_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_48 = \ @@ -3022,7 +3028,7 @@ EXTRA_DIST = \ sndrcv_gzip.sh \ imdtls-basic.sh \ imdtls-basic-tlscommands.sh \ - imdtls-basic-timeout \ + imdtls-basic-timeout.sh \ imdtls-error-cert.sh \ imdtls-sessionbreak.sh \ imdtls-basic-vg.sh \ @@ -3096,6 +3102,7 @@ EXTRA_DIST = \ omhttp-batch-lokirest-retry-vg.sh \ omhttp-retry-vg.sh \ omhttp_server.py \ + omhttp-validate-response.py \ omprog-defaults.sh \ omprog-defaults-vg.sh \ omprog-output-capture.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index c35edb0..73b93e8 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -2476,6 +2476,20 @@ omhttp_get_data() { > ${RSYSLOG_OUT_LOG} } +omhttp_validate_metadata_response() { + echo "starting to validate omhttp response metadata." + omhttp_response_validate_py=$srcdir/omhttp-validate-response.py + if [ ! -f $omhttp_response_validate_py ]; then + echo "Cannot find ${omhttp_response_validate_py} for omhttp test" + error_exit 1 + fi + + $PYTHON ${omhttp_response_validate_py} --error ${RSYSLOG_DYNNAME}/omhttp.error.log --response ${RSYSLOG_DYNNAME}/omhttp.response.log 2>&1 + if [ $? -ne 0 ] ; then + printf 'omhttp_validate_metadata_response failed \n' + error_exit 1 + fi +} # prepare MySQL for next test # each test receives its own database so that we also can run in parallel diff --git a/tests/imdtls-basic-timeout b/tests/imdtls-basic-timeout deleted file mode 100755 index 9b4c46a..0000000 --- a/tests/imdtls-basic-timeout +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# added 2023-10-05 by alorbach -# This file is part of the rsyslog project, released under ASL 2.0 -. ${srcdir:=.}/diag.sh init -export NUMMESSAGES=2000 -export SENDESSAGES=500 -generate_conf -export PORT_RCVR="$(get_free_port)" -export TIMEOUT="5" - -add_conf ' -global( defaultNetstreamDriverCAFile="'$srcdir/tls-certs/ca.pem'" - defaultNetstreamDriverCertFile="'$srcdir/tls-certs/cert.pem'" - defaultNetstreamDriverKeyFile="'$srcdir/tls-certs/key.pem'" -# debug.whitelist="on" -# debug.files=["nsd_ossl.c", "tcpsrv.c", "nsdsel_ossl.c", "nsdpoll_ptcp.c", "dnscache.c"] -) - -module( load="../plugins/imdtls/.libs/imdtls" ) -# tls.authmode="anon" ) -input( type="imdtls" - port="'$PORT_RCVR'" - timeout="'$TIMEOUT'" - ) - -template(name="outfmt" type="string" string="%msg:F,58:2%\n") -:msg, contains, "msgnum:" action( type="omfile" - template="outfmt" - file=`echo $RSYSLOG_OUT_LOG`) -' -# Begin actual testcase - -startup -# valgrind --tool=helgrind $RS_TEST_VALGRIND_EXTRA_OPTS $RS_TESTBENCH_VALGRIND_EXTRA_OPTS --log-fd=1 --error-exitcode=10 -./tcpflood -b1 -W1000 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 -# ./msleep 500 -./tcpflood -b1 -W1000 -i500 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 -./tcpflood -b1 -W1000 -i1000 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 -./tcpflood -b1 -W1000 -i1500 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 - -wait_file_lines -shutdown_when_empty -wait_shutdown -seq_check -exit_test diff --git a/tests/imdtls-basic-timeout.sh b/tests/imdtls-basic-timeout.sh new file mode 100755 index 0000000..9b4c46a --- /dev/null +++ b/tests/imdtls-basic-timeout.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# added 2023-10-05 by alorbach +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +export NUMMESSAGES=2000 +export SENDESSAGES=500 +generate_conf +export PORT_RCVR="$(get_free_port)" +export TIMEOUT="5" + +add_conf ' +global( defaultNetstreamDriverCAFile="'$srcdir/tls-certs/ca.pem'" + defaultNetstreamDriverCertFile="'$srcdir/tls-certs/cert.pem'" + defaultNetstreamDriverKeyFile="'$srcdir/tls-certs/key.pem'" +# debug.whitelist="on" +# debug.files=["nsd_ossl.c", "tcpsrv.c", "nsdsel_ossl.c", "nsdpoll_ptcp.c", "dnscache.c"] +) + +module( load="../plugins/imdtls/.libs/imdtls" ) +# tls.authmode="anon" ) +input( type="imdtls" + port="'$PORT_RCVR'" + timeout="'$TIMEOUT'" + ) + +template(name="outfmt" type="string" string="%msg:F,58:2%\n") +:msg, contains, "msgnum:" action( type="omfile" + template="outfmt" + file=`echo $RSYSLOG_OUT_LOG`) +' +# Begin actual testcase + +startup +# valgrind --tool=helgrind $RS_TEST_VALGRIND_EXTRA_OPTS $RS_TESTBENCH_VALGRIND_EXTRA_OPTS --log-fd=1 --error-exitcode=10 +./tcpflood -b1 -W1000 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 +# ./msleep 500 +./tcpflood -b1 -W1000 -i500 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 +./tcpflood -b1 -W1000 -i1000 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 +./tcpflood -b1 -W1000 -i1500 -p$PORT_RCVR -m$SENDESSAGES -Tdtls -x$srcdir/tls-certs/ca.pem -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem -L0 + +wait_file_lines +shutdown_when_empty +wait_shutdown +seq_check +exit_test diff --git a/tests/omhttp-validate-response.py b/tests/omhttp-validate-response.py new file mode 100644 index 0000000..ef9f09e --- /dev/null +++ b/tests/omhttp-validate-response.py @@ -0,0 +1,34 @@ +import json +import argparse +from collections import defaultdict + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Archive and delete core app log files') + parser.add_argument('--error', action='store', type=str, help='error') + parser.add_argument('--response', action='store', type=str, help='response') + args = parser.parse_args() + + messages = defaultdict(dict) + with open(args.error, "r") as error_f, open(args.response, "r") as response_f: + for line in error_f: + json_obj = json.loads(line) + # postdata contains a json string of records array + records = json.loads(json_obj['request']['postdata']) + if records: + for i, val in enumerate(records['records']): + messages[val['value']['msgnum']]['response'] = json_obj['response'] + messages[val['value']['msgnum']]['index'] = i + #print (len(messages), "messages:", messages) + + # validate with responses + for line in response_f: + json_obj = json.loads(line) + msgnum = json_obj['message']['msgnum'] + code = json_obj['response']['code'] + body = json_obj['response']['body'] + batch_index = json_obj['response']['batch_index'] + #print('msgnum:', msgnum, 'code:', code, 'body:', body, 'batch_index:', batch_index) + assert(msgnum in messages) + assert(messages[msgnum]['response']['status'] == code) + assert(messages[msgnum]['response']['message'] == body) + assert(messages[msgnum]['index'] == batch_index) diff --git a/tests/omhttp_server.py b/tests/omhttp_server.py index 61e0e63..22c7184 100644 --- a/tests/omhttp_server.py +++ b/tests/omhttp_server.py @@ -4,6 +4,8 @@ import json import os import zlib import base64 +import random +import time try: from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer # Python 2 @@ -57,13 +59,27 @@ class MyHandler(BaseHTTPRequestHandler): return if metadata['fail_with_400_after'] != -1 and metadata['posts'] > metadata['fail_with_400_after']: + if metadata['fail_with_delay_secs']: + print("sleeping for: {0}".format(metadata['fail_with_delay_secs'])) + time.sleep(metadata['fail_with_delay_secs']) self.send_response(400) self.end_headers() self.wfile.write(b'BAD REQUEST') return + if metadata['fail_with_401_or_403_after'] != -1 and metadata['posts'] > metadata['fail_with_401_or_403_after']: + status = random.choice([401, 403]) + self.send_response(status) + self.end_headers() + self.wfile.write(b'BAD REQUEST') + return + if metadata['posts'] > 1 and metadata['fail_every'] != -1 and metadata['posts'] % metadata['fail_every'] == 0: - self.send_response(500) + if metadata['fail_with_delay_secs']: + print("sleeping for: {0}".format(metadata['fail_with_delay_secs'])) + time.sleep(metadata['fail_with_delay_secs']) + code = metadata['fail_with'] if metadata['fail_with'] else 500 + self.send_response(code) self.end_headers() self.wfile.write(b'INTERNAL ERROR') return @@ -114,13 +130,19 @@ if __name__ == '__main__': parser.add_argument('-i', '--interface', action='store', type=str, default='localhost', help='port') parser.add_argument('--fail-after', action='store', type=int, default=0, help='start failing after n posts') parser.add_argument('--fail-every', action='store', type=int, default=-1, help='fail every n posts') + parser.add_argument('--fail-with', action='store', type=int, default=500, help='on failure, fail with this code') parser.add_argument('--fail-with-400-after', action='store', type=int, default=-1, help='fail with 400 after n posts') + parser.add_argument('--fail-with-401-or-403-after', action='store', type=int, default=-1, help='fail with 401 or 403 after n posts') + parser.add_argument('--fail-with-delay-secs', action='store', type=int, default=0, help='fail with n secs of delay') parser.add_argument('--decompress', action='store_true', default=False, help='decompress posted data') parser.add_argument('--userpwd', action='store', default='', help='only accept this user:password combination') args = parser.parse_args() metadata['fail_after'] = args.fail_after metadata['fail_every'] = args.fail_every + metadata['fail_with'] = args.fail_with metadata['fail_with_400_after'] = args.fail_with_400_after + metadata['fail_with_401_or_403_after'] = args.fail_with_401_or_403_after + metadata['fail_with_delay_secs'] = args.fail_with_delay_secs metadata['decompress'] = args.decompress metadata['userpwd'] = args.userpwd server = HTTPServer((args.interface, args.port), MyHandler) diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index d27a2a7..55497ac 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -1662,7 +1662,6 @@ initAll(int argc, char **argv) CAP_FIELD(CAP_BLOCK_SUSPEND, CAPNG_EFFECTIVE | CAPNG_PERMITTED), CAP_FIELD(CAP_NET_RAW, CAPNG_EFFECTIVE | CAPNG_PERMITTED ), CAP_FIELD(CAP_CHOWN, CAPNG_EFFECTIVE | CAPNG_PERMITTED ), - CAP_FIELD(CAP_IPC_LOCK, CAPNG_EFFECTIVE | CAPNG_PERMITTED ), CAP_FIELD(CAP_LEASE, CAPNG_EFFECTIVE | CAPNG_PERMITTED), CAP_FIELD(CAP_NET_ADMIN, CAPNG_EFFECTIVE | CAPNG_PERMITTED), CAP_FIELD(CAP_NET_BIND_SERVICE, CAPNG_EFFECTIVE | CAPNG_PERMITTED), -- cgit v1.2.3