diff options
Diffstat (limited to '')
| -rw-r--r-- | plugins/omclickhouse/omclickhouse.c | 1032 |
1 files changed, 1032 insertions, 0 deletions
diff --git a/plugins/omclickhouse/omclickhouse.c b/plugins/omclickhouse/omclickhouse.c new file mode 100644 index 0000000..2c67e5e --- /dev/null +++ b/plugins/omclickhouse/omclickhouse.c @@ -0,0 +1,1032 @@ +/* omclickhouse.c + * This is the https://clickhouse.yandex/ output module. + * + * Copyright 2018 Pascal Withopf and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <memory.h> +#include <string.h> +#include <curl/curl.h> +#include <curl/easy.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#if defined(__FreeBSD__) +#include <unistd.h> +#endif +#include <json.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "statsobj.h" +#include "cfsysline.h" +#include "unicode-helper.h" +#include "obj-types.h" +#include "ratelimit.h" +#include "ruleset.h" + +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP +MODULE_CNFNAME("omclickhouse") + +/* internal structures */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(statsobj) +DEFobjCurrIf(prop) +DEFobjCurrIf(ruleset) + +statsobj_t *indexStats; +STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit) +STATSCOUNTER_DEF(indexHTTPFail, mutIndexHTTPFail) +STATSCOUNTER_DEF(indexHTTPReqFail, mutIndexHTTPReqFail) +STATSCOUNTER_DEF(indexFail, mutIndexFail) +STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) + + + +typedef struct curl_slist HEADER; +typedef struct instanceConf_s { + uchar *serverBaseUrl; + int port; + uchar *user; + uchar *pwd; + long healthCheckTimeout; + long timeout; + uchar *authBuf; + uchar *tplName; + sbool useHttps; + sbool allowUnsignedCerts; + sbool skipVerifyHost; + int fdErrFile; + uchar *errorFile; + sbool bulkmode; + size_t maxbytes; + uchar *caCertFile; + uchar *myCertFile; + uchar *myPrivKeyFile; + struct instanceConf_s *next; +} instanceData; + +struct modConfData_s { + rsconf_t *pConf; /* our overall config object */ + instanceConf_t *root, *tail; +}; +static modConfData_t *loadModConf = NULL; /* modConf ptr to use for the current load process */ + +typedef struct wrkrInstanceData { + PTR_ASSERT_DEF + instanceData *pData; + CURL *curlPostHandle; /* libcurl session handle for posting data to the server */ + HEADER *curlHeader; /* json POST request info */ + CURL *curlCheckConnHandle; /* libcurl session handle for checking the server connection */ + int replyLen; + char *reply; + uchar *restURL; + struct { + es_str_t *data; + int nmemb; /* number of messages in batch (for statistics counting) */ + } batch; + sbool insertErrorSent; /* needed for insert error message */ +} wrkrInstanceData_t; + +/* tables for interfacing with the v6 config system */ +/* action (instance) parameters */ +static struct cnfparamdescr actpdescr[] = { + { "server", eCmdHdlrGetWord, 0 }, + { "port", eCmdHdlrInt, 0 }, + { "user", eCmdHdlrGetWord, 0 }, + { "pwd", eCmdHdlrGetWord, 0 }, + { "healthchecktimeout", eCmdHdlrInt, 0 }, + { "timeout", eCmdHdlrInt, 0 }, + { "template", eCmdHdlrGetWord, 0 }, + { "usehttps", eCmdHdlrBinary, 0 }, + { "allowunsignedcerts", eCmdHdlrBinary, 0 }, + { "skipverifyhost", eCmdHdlrBinary, 0 }, + { "errorfile", eCmdHdlrGetWord, 0 }, + { "bulkmode", eCmdHdlrBinary, 0 }, + { "maxbytes", eCmdHdlrSize, 0 }, + { "tls.cacert", eCmdHdlrString, 0 }, + { "tls.mycert", eCmdHdlrString, 0 }, + { "tls.myprivkey", eCmdHdlrString, 0 } +}; +static struct cnfparamblk actpblk = + { CNFPARAMBLK_VERSION, + sizeof(actpdescr)/sizeof(struct cnfparamdescr), + actpdescr + }; + +static rsRetVal curlSetup(wrkrInstanceData_t *pWrkrData); + +BEGINcreateInstance +CODESTARTcreateInstance + pData->fdErrFile = -1; + pData->caCertFile = NULL; + pData->myCertFile = NULL; + pData->myPrivKeyFile = NULL; +ENDcreateInstance + +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->curlHeader = NULL; + pWrkrData->curlPostHandle = NULL; + pWrkrData->curlCheckConnHandle = NULL; + pWrkrData->restURL = NULL; + if(pData->bulkmode) { + if((pWrkrData->batch.data = es_newStr(1024)) == NULL) { + LogError(0, RS_RET_OUT_OF_MEMORY, + "omclickhouse: error creating batch string " + "turned off bulk mode\n"); + pData->bulkmode = 0; /* at least it works */ + } + } + pWrkrData->insertErrorSent = 0; + + iRet = curlSetup(pWrkrData); +ENDcreateWrkrInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + free(pData->serverBaseUrl); + free(pData->user); + free(pData->pwd); + free(pData->authBuf); + if(pData->fdErrFile != -1) + close(pData->fdErrFile); + free(pData->errorFile); + free(pData->tplName); + free(pData->caCertFile); + free(pData->myCertFile); + free(pData->myPrivKeyFile); +ENDfreeInstance + +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + if(pWrkrData->curlHeader != NULL) { + curl_slist_free_all(pWrkrData->curlHeader); + pWrkrData->curlHeader = NULL; + } + if(pWrkrData->curlCheckConnHandle != NULL) { + curl_easy_cleanup(pWrkrData->curlCheckConnHandle); + pWrkrData->curlCheckConnHandle = NULL; + } + if(pWrkrData->curlPostHandle != NULL) { + curl_easy_cleanup(pWrkrData->curlPostHandle); + pWrkrData->curlPostHandle = NULL; + } + if (pWrkrData->restURL != NULL) { + free(pWrkrData->restURL); + pWrkrData->restURL = NULL; + } + es_deleteStr(pWrkrData->batch.data); +ENDfreeWrkrInstance + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + dbgprintf("omclickhouse\n"); + dbgprintf("\tserver='%s'\n", pData->serverBaseUrl); + dbgprintf("\tport='%d'\n", pData->port); + dbgprintf("\tuser='%s'\n", pData->user); + dbgprintf("\tpwd='%s'\n", pData->pwd); + dbgprintf("\thealthCheckTimeout=%lu\n", pData->healthCheckTimeout); + dbgprintf("\ttimeout=%lu\n", pData->timeout); + dbgprintf("\ttemplate='%s'\n", pData->tplName); + dbgprintf("\tusehttps='%d'\n", pData->useHttps); + dbgprintf("\tallowunsignedcerts='%d'\n", pData->allowUnsignedCerts); + dbgprintf("\tskipverifyhost='%d'\n", pData->skipVerifyHost); + dbgprintf("\terrorFile='%s'\n", pData->errorFile); + dbgprintf("\tbulkmode='%d'\n", pData->bulkmode); + dbgprintf("\tmaxbytes='%zu'\n", pData->maxbytes); + dbgprintf("\ttls.cacert='%s'\n", pData->caCertFile); + dbgprintf("\ttls.mycert='%s'\n", pData->myCertFile); + dbgprintf("\ttls.myprivkey='%s'\n", pData->myPrivKeyFile); +ENDdbgPrintInstInfo + + +/* checks if connection to clickhouse can be established + */ +static rsRetVal ATTR_NONNULL() +checkConn(wrkrInstanceData_t *const pWrkrData) +{ + CURL *curl; + CURLcode res; + char errbuf[CURL_ERROR_SIZE] = ""; + const char* healthCheckMessage ="SELECT 1"; + DEFiRet; + + pWrkrData->reply = NULL; + pWrkrData->replyLen = 0; + curl = pWrkrData->curlCheckConnHandle; + + + curl_easy_setopt(curl, CURLOPT_URL, pWrkrData->restURL); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, healthCheckMessage); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(healthCheckMessage)); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + res = curl_easy_perform(curl); + + if (res == CURLE_OK) { + DBGPRINTF("omclickhouse: checkConn completed with success\n"); + ABORT_FINALIZE(RS_RET_OK); + } + + DBGPRINTF("omclickhouse: checkConn failed: %s\n", + curl_easy_strerror(res)); + + LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING, + "omclickhouse: checkConn failed."); + ABORT_FINALIZE(RS_RET_SUSPENDED); + +finalize_it: + free(pWrkrData->reply); + pWrkrData->reply = NULL; /* don't leave dangling pointer */ + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + dbgprintf("omclickhouse: tryResume called\n"); + iRet = checkConn(pWrkrData); +ENDtryResume + + +/* + * Dumps entire bulk request and response in error log + */ +static rsRetVal +getDataErrorDefault(wrkrInstanceData_t *pWrkrData, char *reply, uchar *reqmsg, char **rendered) +{ + DEFiRet; + fjson_object *req=NULL; + fjson_object *errRoot=NULL; + + if((req=fjson_object_new_object()) == NULL) ABORT_FINALIZE(RS_RET_ERR); + fjson_object_object_add(req, "url", fjson_object_new_string((char*)pWrkrData->restURL)); + fjson_object_object_add(req, "postdata", fjson_object_new_string((char*)reqmsg)); + + if((errRoot=fjson_object_new_object()) == NULL) ABORT_FINALIZE(RS_RET_ERR); + fjson_object_object_add(errRoot, "request", req); + fjson_object_object_add(errRoot, "reply", fjson_object_new_string(reply)); + *rendered = strdup((char*)fjson_object_to_json_string(errRoot)); + + req=NULL; + fjson_object_put(errRoot); + + finalize_it: + fjson_object_put(req); + RETiRet; +} + + +/* write data error request/replies to separate error file + * Note: we open the file but never close it before exit. If it + * needs to be closed, HUP must be sent. + */ +static rsRetVal ATTR_NONNULL() +writeDataError(wrkrInstanceData_t *const pWrkrData, uchar *const reqmsg) +{ + DEFiRet; + instanceData *pData = pWrkrData->pData; + char *rendered = pWrkrData->reply; + size_t toWrite; + ssize_t wrRet; + + if(pData->errorFile == NULL) { + dbgprintf("omclickhouse: no local error logger defined - " + "ignoring ClickHouse error information\n"); + FINALIZE; + } + + + if(pData->fdErrFile == -1) { + pData->fdErrFile = open((char*)pData->errorFile, + O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE|O_CLOEXEC, + S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); + if(pData->fdErrFile == -1) { + LogError(errno, RS_RET_ERR, "omclickhouse: error opening error file %s", + pData->errorFile); + ABORT_FINALIZE(RS_RET_ERR); + } + } + + if(getDataErrorDefault(pWrkrData, pWrkrData->reply, reqmsg, &rendered) != RS_RET_OK) { + ABORT_FINALIZE(RS_RET_ERR); + } + + /* we do not do real error-handling on the err file, as this finally complicates + * things way to much. + */ + dbgprintf("omclickhouse: message sent: '%s'\n", reqmsg); + dbgprintf("omclickhouse: error record: '%s'\n", rendered); + toWrite = strlen(rendered) + 1; + /* Note: we overwrite the '\0' terminator with '\n' -- so we avoid + * caling malloc() -- write() does NOT need '\0'! + */ + rendered[toWrite-1] = '\n'; /* NO LONGER A STRING! */ + wrRet = write(pData->fdErrFile, rendered, toWrite); + if(wrRet != (ssize_t) toWrite) { + LogError(errno, RS_RET_IO_ERROR, + "omclickhouse: error writing error file %s, write returned %lld", + pData->errorFile, (long long) wrRet); + } + +finalize_it: + RETiRet; +} + + +static rsRetVal +checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) +{ + DEFiRet; + + if((strstr(pWrkrData->reply, " = DB::Exception" ) != NULL) + || (strstr(pWrkrData->reply, "DB::NetException" ) != NULL) + || (strstr(pWrkrData->reply, "DB::ParsingException" ) != NULL)) { + dbgprintf("omclickhouse: action failed with error: %s\n", pWrkrData->reply); + iRet = RS_RET_DATAFAIL; + } + + if(iRet == RS_RET_DATAFAIL) { + STATSCOUNTER_INC(indexFail, mutIndexFail); + writeDataError(pWrkrData, reqmsg); + iRet = RS_RET_OK; /* we have handled the problem! */ + } + + + if(iRet != RS_RET_OK) { + STATSCOUNTER_INC(indexFail, mutIndexFail); + } + RETiRet; +} + +static rsRetVal ATTR_NONNULL(1) +setPostURL(wrkrInstanceData_t *const pWrkrData) +{ + char* baseUrl; + es_str_t *url; + DEFiRet; + instanceData *const pData = pWrkrData->pData; + + baseUrl = (char*)pData->serverBaseUrl; + url = es_newStrFromCStr(baseUrl, strlen(baseUrl)); + if (url == NULL) { + LogError(0, RS_RET_OUT_OF_MEMORY, + "omclickhouse: error allocating new estr for POST url."); + ABORT_FINALIZE(RS_RET_ERR); + } + + if(pWrkrData->restURL != NULL) + free(pWrkrData->restURL); + + pWrkrData->restURL = (uchar*)es_str2cstr(url, NULL); + curl_easy_setopt(pWrkrData->curlPostHandle, CURLOPT_URL, pWrkrData->restURL); + dbgprintf("omclickhouse: using REST URL: '%s'\n", pWrkrData->restURL); + +finalize_it: + if (url != NULL) + es_deleteStr(url); + RETiRet; +} + + +/* this method computes the next data set to be added to the batch + * returns the expected size of adding the next message into the + * batched request to clickhouse + */ +static size_t +computeBulkMessage(const wrkrInstanceData_t *const pWrkrData, + const uchar *const message, char **newMessage) +{ + size_t r = 0; + char *v; + if (pWrkrData->batch.nmemb != 0 + && (v = strstr((const char *)message, "VALUES")) != NULL + && (v = strchr(v, '(')) != NULL + ) { + *newMessage = v; + r = strlen(*newMessage); + } else { + *newMessage = (char*)message; + r = strlen(*newMessage); + } + dbgprintf("omclickhouse: computeBulkMessage: new message part: %s\n", *newMessage); + + return r; +} + + +/* This method builds the batch, that will be submitted. + */ +static rsRetVal +buildBatch(wrkrInstanceData_t *pWrkrData, char *message) +{ + DEFiRet; + int length = strlen(message); + int r; + + r = es_addBuf(&pWrkrData->batch.data, message, length); + if(r != 0) { + LogError(0, RS_RET_ERR, "omclickhouse: growing batch failed with code %d", r); + ABORT_FINALIZE(RS_RET_ERR); + } + ++pWrkrData->batch.nmemb; + iRet = RS_RET_OK; + +finalize_it: + RETiRet; +} + + +static void ATTR_NONNULL() +initializeBatch(wrkrInstanceData_t *pWrkrData) +{ + es_emptyStr(pWrkrData->batch.data); + pWrkrData->batch.nmemb = 0; +} + + +static rsRetVal ATTR_NONNULL(1, 2) +curlPost(wrkrInstanceData_t *pWrkrData, uchar *message, int msglen, const int nmsgs) +{ + CURLcode code; + CURL *const curl = pWrkrData->curlPostHandle; + char errbuf[CURL_ERROR_SIZE] = ""; + DEFiRet; + + if(!strstr((char*)message, "INSERT INTO") && !pWrkrData->insertErrorSent) { + indexHTTPFail += nmsgs; + LogError(0, RS_RET_ERR, "omclickhouse: Message is no Insert query: " + "Message suspended: %s", (char*)message); + pWrkrData->insertErrorSent = 1; + ABORT_FINALIZE(RS_RET_ERR); + } + + pWrkrData->reply = NULL; + pWrkrData->replyLen = 0; + + CHKiRet(setPostURL(pWrkrData)); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, msglen); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + code = curl_easy_perform(curl); + dbgprintf("curl returned %lld\n", (long long) code); + if (code != CURLE_OK && code != CURLE_HTTP_RETURNED_ERROR) { + STATSCOUNTER_INC(indexHTTPReqFail, mutIndexHTTPReqFail); + indexHTTPFail += nmsgs; + LogError(0, RS_RET_SUSPENDED, + "omclickhouse: we are suspending ourselfs due " + "to server failure %lld: %s", (long long) code, errbuf); + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + + if(pWrkrData->reply == NULL) { + dbgprintf("omclickhouse: pWrkrData reply==NULL, replyLen = '%d'\n", + pWrkrData->replyLen); + STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); + } else { + dbgprintf("omclickhouse: pWrkrData replyLen = '%d'\n", pWrkrData->replyLen); + if(pWrkrData->replyLen > 0) { + pWrkrData->reply[pWrkrData->replyLen] = '\0'; + /* Append 0 Byte if replyLen is above 0 - byte has been reserved in malloc */ + } + dbgprintf("omclickhouse: pWrkrData reply: '%s'\n", pWrkrData->reply); + CHKiRet(checkResult(pWrkrData, message)); + } + +finalize_it: + free(pWrkrData->reply); + pWrkrData->reply = NULL; /* don't leave dangling pointer */ + RETiRet; +} + + +static rsRetVal +submitBatch(wrkrInstanceData_t *pWrkrData) +{ + char *cstr = NULL; + DEFiRet; + + cstr = es_str2cstr(pWrkrData->batch.data, NULL); + dbgprintf("omclickhouse: submitBatch, batch: '%s'\n", cstr); + + CHKiRet(curlPost(pWrkrData, (uchar*) cstr, strlen(cstr), pWrkrData->batch.nmemb)); + +finalize_it: + free(cstr); + RETiRet; +} + + +BEGINbeginTransaction +CODESTARTbeginTransaction + if(!pWrkrData->pData->bulkmode) { + FINALIZE; + } + + initializeBatch(pWrkrData); +finalize_it: +ENDbeginTransaction + + +BEGINdoAction + char *batchPart = NULL; +CODESTARTdoAction + dbgprintf("CODESTARTdoAction: entered\n"); + STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); + + if(pWrkrData->pData->bulkmode) { + const size_t nBytes = computeBulkMessage(pWrkrData, ppString[0], &batchPart); + dbgprintf("pascal: doAction: message: %s\n", batchPart); + + /* If max bytes is set and this next message will put us over the limit, + * submit the current buffer and reset */ + if(pWrkrData->pData->maxbytes > 0 + && es_strlen(pWrkrData->batch.data) + nBytes > pWrkrData->pData->maxbytes) { + + dbgprintf("omclickhouse: maxbytes limit reached, submitting partial " + "batch of %d elements.\n", pWrkrData->batch.nmemb); + CHKiRet(submitBatch(pWrkrData)); + initializeBatch(pWrkrData); + batchPart = (char*)ppString[0]; + } + + CHKiRet(buildBatch(pWrkrData, batchPart)); + + iRet = pWrkrData->batch.nmemb == 1 ? RS_RET_PREVIOUS_COMMITTED : RS_RET_DEFER_COMMIT; + } else { + CHKiRet(curlPost(pWrkrData, ppString[0], strlen((char*)ppString[0]), 1)); + } +finalize_it: +ENDdoAction + + +BEGINendTransaction +CODESTARTendTransaction +/* End Transaction only if batch data is not empty */ + if (pWrkrData->batch.data != NULL && pWrkrData->batch.nmemb > 0) { + CHKiRet(submitBatch(pWrkrData)); + } else { + dbgprintf("omclickhouse: endTransaction, pWrkrData->batch.data is NULL, " + "nothing to send. \n"); + } +finalize_it: +ENDendTransaction + +static void ATTR_NONNULL() +setInstParamDefaults(instanceData *const pData) +{ + pData->serverBaseUrl = NULL; + pData->port = 8123; + pData->user = NULL; + pData->pwd = NULL; + pData->healthCheckTimeout = 3500; + pData->timeout = 0; + pData->authBuf = NULL; + pData->tplName = NULL; + pData->useHttps = 1; + pData->allowUnsignedCerts = 1; + pData->skipVerifyHost = 0; + pData->errorFile = NULL; + pData->bulkmode = 1; + pData->maxbytes = 104857600; //100MB + pData->caCertFile = NULL; + pData->myCertFile = NULL; + pData->myPrivKeyFile = NULL; +} + +/* POST result string ... useful for debugging */ +static size_t +curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + char *p = (char *)ptr; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t*) userdata; + char *buf; + size_t newlen; + newlen = pWrkrData->replyLen + size*nmemb; + if((buf = realloc(pWrkrData->reply, newlen + 1)) == NULL) { + LogError(errno, RS_RET_ERR, "omclickhouse: realloc failed in curlResult"); + return 0; /* abort due to failure */ + } + memcpy(buf+pWrkrData->replyLen, p, size*nmemb); + pWrkrData->replyLen = newlen; + pWrkrData->reply = buf; + return size*nmemb; +} + +static void ATTR_NONNULL() +curlSetupCommon(wrkrInstanceData_t *const pWrkrData, CURL *const handle) +{ + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, pWrkrData->curlHeader); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, TRUE); + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); + curl_easy_setopt(handle, CURLOPT_WRITEDATA, pWrkrData); + if(pWrkrData->pData->allowUnsignedCerts) + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, FALSE); + if(pWrkrData->pData->skipVerifyHost) + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, FALSE); + if(pWrkrData->pData->authBuf != NULL) { + curl_easy_setopt(handle, CURLOPT_USERPWD, pWrkrData->pData->authBuf); + curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + } + + if(pWrkrData->pData->caCertFile) { + curl_easy_setopt(handle, CURLOPT_CAINFO, pWrkrData->pData->caCertFile); + } + if(pWrkrData->pData->myCertFile) { + curl_easy_setopt(handle, CURLOPT_SSLCERT, pWrkrData->pData->myCertFile); + } + if(pWrkrData->pData->myPrivKeyFile) { + curl_easy_setopt(handle, CURLOPT_SSLKEY, pWrkrData->pData->myPrivKeyFile); + } + /* uncomment for in-dept debuggung: + curl_easy_setopt(handle, CURLOPT_VERBOSE, TRUE); */ +} + + +static void ATTR_NONNULL() +curlCheckConnSetup(wrkrInstanceData_t *const pWrkrData) +{ + curlSetupCommon(pWrkrData, pWrkrData->curlCheckConnHandle); + curl_easy_setopt(pWrkrData->curlCheckConnHandle, + CURLOPT_TIMEOUT_MS, pWrkrData->pData->healthCheckTimeout); +} + + +static void ATTR_NONNULL(1) +curlPostSetup(wrkrInstanceData_t *const pWrkrData) +{ + curlSetupCommon(pWrkrData, pWrkrData->curlPostHandle); + curl_easy_setopt(pWrkrData->curlPostHandle, CURLOPT_POST, 1); + if(pWrkrData->pData->timeout) { + curl_easy_setopt(pWrkrData->curlPostHandle, + CURLOPT_TIMEOUT_MS, pWrkrData->pData->timeout); + } +} + +#define CONTENT_JSON "Content-Type: application/json; charset=utf-8" + +static rsRetVal ATTR_NONNULL() +curlSetup(wrkrInstanceData_t *const pWrkrData) +{ + DEFiRet; + pWrkrData->curlHeader = curl_slist_append(NULL, CONTENT_JSON); + CHKmalloc(pWrkrData->curlPostHandle = curl_easy_init()); + curlPostSetup(pWrkrData); + + CHKmalloc(pWrkrData->curlCheckConnHandle = curl_easy_init()); + curlCheckConnSetup(pWrkrData); + +finalize_it: + if(iRet != RS_RET_OK && pWrkrData->curlPostHandle != NULL) { + curl_easy_cleanup(pWrkrData->curlPostHandle); + pWrkrData->curlPostHandle = NULL; + } + RETiRet; +} + +static rsRetVal +computeAuthHeader(char* user, char* pwd, uchar** authBuf) +{ + DEFiRet; + int r; + + es_str_t* auth = es_newStr(1024); + if (auth == NULL) { + LogError(0, RS_RET_OUT_OF_MEMORY, + "omclickhouse: failed to allocate es_str auth for auth header construction"); + ABORT_FINALIZE(RS_RET_ERR); + } + + r = es_addBuf(&auth, user, strlen(user)); + if(r == 0) + r = es_addChar(&auth, ':'); + if(r == 0 && pwd != NULL) + r = es_addBuf(&auth, pwd, strlen(pwd)); + if(r == 0) + *authBuf = (uchar*) es_str2cstr(auth, NULL); + + if (r != 0 || *authBuf == NULL) { + LogError(0, RS_RET_ERR, "omclickhouse: failed to build auth header\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + if (auth != NULL) + es_deleteStr(auth); + RETiRet; +} + +/* Build basic URL part, which includes hostname, port, user, pwd as follows: + * http://user:pwd@hostname:port/ based on a server param + * Newly creates a cstr for this purpose. + */ +static rsRetVal +computeBaseUrl(const char* server, const int port, const sbool useHttps, instanceData *pData) +{ +# define SCHEME_HTTPS "https://" +# define SCHEME_HTTP "http://" + + char portBuf[64]; + int r = 0; + const char *host = server; + DEFiRet; + + assert(server[strlen(server)-1] != '/'); + + es_str_t *urlBuf = es_newStr(256); + if (urlBuf == NULL) { + LogError(0, RS_RET_OUT_OF_MEMORY, + "omclickhouse: failed to allocate es_str urlBuf in computeBaseUrl"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* Find where the hostname/ip of the server starts. If the scheme is not specified + in the uri, start the buffer with a scheme corresponding to the useHttps parameter. + */ + if(strcasestr(server, SCHEME_HTTP)) { + host = server + strlen(SCHEME_HTTP); + } else if(strcasestr(server, SCHEME_HTTPS)) { + host = server + strlen(SCHEME_HTTPS); + } else { + r = useHttps ? es_addBuf(&urlBuf, SCHEME_HTTPS, sizeof(SCHEME_HTTPS)-1) : + es_addBuf(&urlBuf, SCHEME_HTTP, sizeof(SCHEME_HTTP)-1); + } + if (r == 0) + r = es_addBuf(&urlBuf, (char *)server, strlen(server)); + if (r == 0 && !strchr(host, ':')) { + snprintf(portBuf, sizeof(portBuf), ":%d", port); + r = es_addBuf(&urlBuf, portBuf, strlen(portBuf)); + } + if (r == 0) + r = es_addChar(&urlBuf, '/'); + if (r == 0) + pData->serverBaseUrl = (uchar*) es_str2cstr(urlBuf, NULL); + + if (r != 0 || pData->serverBaseUrl == NULL) { + LogError(0, RS_RET_ERR, "omclickhouse: error occurred computing baseUrl from " + "server %s", server); + ABORT_FINALIZE(RS_RET_ERR); + } +finalize_it: + if (urlBuf) { + es_deleteStr(urlBuf); + } + RETiRet; +} + +BEGINnewActInst + struct cnfparamvals *pvals; + uchar *server = NULL; + int i; + FILE *fp; + char errStr[1024]; +CODESTARTnewActInst + if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + CHKiRet(createInstance(&pData)); + setInstParamDefaults(pData); + + for(i = 0 ; i < actpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(actpblk.descr[i].name, "server")) { + server = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "port")) { + pData->port = (int) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "user")) { + pData->user = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "pwd")) { + pData->pwd = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "healthchecktimeout")) { + pData->healthCheckTimeout = (long) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "timeout")) { + pData->timeout = (long) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "template")) { + pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "usehttps")) { + pData->useHttps = pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "allowunsignedcerts")) { + pData->allowUnsignedCerts = pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "skipverifyhost")) { + pData->skipVerifyHost = pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "errorfile")) { + pData->errorFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(actpblk.descr[i].name, "bulkmode")) { + pData->bulkmode = pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "maxbytes")) { + pData->maxbytes = (size_t) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "tls.cacert")) { + pData->caCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->caCertFile, "r"); + if(fp == NULL) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + LogError(0, RS_RET_NO_FILE_ACCESS, + "error: omclickhouse: 'tls.cacert' file %s couldn't be accessed: %s\n", + pData->caCertFile, errStr); + } else { + fclose(fp); + } + } else if(!strcmp(actpblk.descr[i].name, "tls.mycert")) { + pData->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->myCertFile, "r"); + if(fp == NULL) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + LogError(0, RS_RET_NO_FILE_ACCESS, + "error: omclickhouse: 'tls.mycert' file %s couldn't be accessed: %s\n", + pData->myCertFile, errStr); + } else { + fclose(fp); + } + } else if(!strcmp(actpblk.descr[i].name, "tls.myprivkey")) { + pData->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->myPrivKeyFile, "r"); + if(fp == NULL) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + LogError(0, RS_RET_NO_FILE_ACCESS, + "error: omclickhouse: 'tls.myprivkey' file %s couldn't be accessed: %s\n", + pData->myPrivKeyFile, errStr); + } else { + fclose(fp); + } + } else { + LogError(0, RS_RET_INTERNAL_ERROR, "omclickhouse: program error, " + "non-handled param '%s'", actpblk.descr[i].name); + } + } + + + if(pData->user == NULL && pData->pwd != NULL) { + LogMsg(0, RS_RET_OK, LOG_WARNING, "omclickhouse: No user was specified " + "but a password was given."); + } + + if(pData->user != NULL) + CHKiRet(computeAuthHeader((char*) pData->user, (char*) pData->pwd, &pData->authBuf)); + + CODE_STD_STRING_REQUESTnewActInst(1) + CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*)strdup((pData->tplName == NULL) ? + " StdClickHouseFmt" : (char*)pData->tplName), OMSR_RQD_TPL_OPT_SQL)); + + if(server != NULL) { + CHKiRet(computeBaseUrl((const char*)server, pData->port, pData->useHttps, pData)); + } else { + LogMsg(0, RS_RET_OK, LOG_WARNING, + "omclickhouse: No servers specified, using localhost"); + CHKiRet(computeBaseUrl("localhost", pData->port, pData->useHttps, + pData)); + } + + /* node created, let's add to list of instance configs for the module */ + if(loadModConf->tail == NULL) { + loadModConf->tail = loadModConf->root = pData; + } else { + loadModConf->tail->next = pData; + loadModConf->tail = pData; + } + +CODE_STD_FINALIZERnewActInst + free(server); + cnfparamvalsDestruct(pvals, &actpblk); +ENDnewActInst + + +BEGINbeginCnfLoad +CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; + pModConf->root = pModConf->tail = NULL; +ENDbeginCnfLoad + + +BEGINendCnfLoad +CODESTARTendCnfLoad + loadModConf = NULL; /* done loading */ +ENDendCnfLoad + + +BEGINcheckCnf +CODESTARTcheckCnf +ENDcheckCnf + + +BEGINactivateCnf +CODESTARTactivateCnf +ENDactivateCnf + + +BEGINfreeCnf +CODESTARTfreeCnf +ENDfreeCnf + + +BEGINdoHUP +CODESTARTdoHUP + if(pData->fdErrFile != -1) { + close(pData->fdErrFile); + pData->fdErrFile = -1; + } +ENDdoHUP + + +BEGINmodExit +CODESTARTmodExit + curl_global_cleanup(); + statsobj.Destruct(&indexStats); + objRelease(statsobj, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDmodExit + +NO_LEGACY_CONF_parseSelectorAct + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_doHUP +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ +CODEqueryEtryPt_STD_CONF2_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(statsobj, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -indexing disabled"); + ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED); + } + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&indexStats)); + CHKiRet(statsobj.SetName(indexStats, (uchar *)"omclickhouse")); + CHKiRet(statsobj.SetOrigin(indexStats, (uchar *)"omclickhouse")); + STATSCOUNTER_INIT(indexSubmit, mutIndexSubmit); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submitted", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexSubmit)); + STATSCOUNTER_INIT(indexHTTPFail, mutIndexHTTPFail); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.http", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexHTTPFail)); + STATSCOUNTER_INIT(indexHTTPReqFail, mutIndexHTTPReqFail); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.httprequests", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexHTTPReqFail)); + STATSCOUNTER_INIT(indexFail, mutIndexFail); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.clickhouse", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexFail)); + STATSCOUNTER_INIT(indexSuccess, mutIndexSuccess); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"response.success", + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexSuccess)); + CHKiRet(statsobj.ConstructFinalize(indexStats)); + +ENDmodInit + +/* vi:set ai: + */ |