diff options
Diffstat (limited to 'src')
43 files changed, 541 insertions, 237 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index d831700..99e40c4 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1281,7 +1281,8 @@ EXTRA_DIST = \ tests/detect-tls-version.c \ tests/detect-ipaddr.c \ tests/detect.c \ - tests/stream-tcp.c + tests/stream-tcp.c \ + tests/output-json-stats.c install-headers: mkdir -p $(DESTDIR)${includedir}/suricata diff --git a/src/Makefile.in b/src/Makefile.in index 7317ef0..4f15923 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -2600,7 +2600,8 @@ EXTRA_DIST = \ tests/detect-tls-version.c \ tests/detect-ipaddr.c \ tests/detect.c \ - tests/stream-tcp.c + tests/stream-tcp.c \ + tests/output-json-stats.c # set the include path found by configure diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 7783c07..e9b84ed 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -1444,7 +1444,6 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow /* set the packets to no inspection and reassembly if required */ if (pstate->flags & APP_LAYER_PARSER_NO_INSPECTION) { AppLayerParserSetEOF(pstate); - FlowSetNoPayloadInspectionFlag(f); if (f->proto == IPPROTO_TCP) { StreamTcpDisableAppLayer(f); @@ -1466,6 +1465,9 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow StreamTcpSetSessionBypassFlag(ssn); } } + } else { + // for TCP, this is set after flushing + FlowSetNoPayloadInspectionFlag(f); } } diff --git a/src/autoconf.h b/src/autoconf.h index 8c42d6c..336c550 100644 --- a/src/autoconf.h +++ b/src/autoconf.h @@ -696,7 +696,7 @@ #define PACKAGE_NAME "suricata" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "suricata 7.0.3" +#define PACKAGE_STRING "suricata 7.0.4" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "suricata" @@ -705,7 +705,7 @@ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "7.0.3" +#define PACKAGE_VERSION "7.0.4" /* Pcre code unit width is 8 bits */ #define PCRE2_CODE_UNIT_WIDTH 8 @@ -723,7 +723,7 @@ /* #undef PROFILING */ /* Git revision */ -#define REVISION be68bbc4a 2024-02-08 +#define REVISION d8bad3b1a 2024-03-19 /* Define to 1 if all of the C90 standard headers exist (not just the ones required in a freestanding environment). This macro is provided for @@ -743,7 +743,7 @@ /* #undef UNITTESTS */ /* Version number of package */ -#define VERSION "7.0.3" +#define VERSION "7.0.4" /* Enable Windows WinDivert support for inline IDP */ /* #undef WINDIVERT */ diff --git a/src/conf-yaml-loader.c b/src/conf-yaml-loader.c index 1bd107e..ea64563 100644 --- a/src/conf-yaml-loader.c +++ b/src/conf-yaml-loader.c @@ -185,7 +185,7 @@ static int ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq, int while (!done) { if (!yaml_parser_parse(parser, &event)) { - SCLogError("Failed to parse configuration file at line %" PRIuMAX ": %s\n", + SCLogError("Failed to parse configuration file at line %" PRIuMAX ": %s", (uintmax_t)parser->problem_mark.line, parser->problem); retval = -1; break; diff --git a/src/decode-pppoe.c b/src/decode-pppoe.c index f884085..eb5e6ac 100644 --- a/src/decode-pppoe.c +++ b/src/decode-pppoe.c @@ -80,11 +80,6 @@ int DecodePPPOEDiscovery(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, return TM_ECODE_OK; } - /* parse any tags we have in the packet */ - - uint32_t tag_length = 0; - PPPOEDiscoveryTag* pppoedt = (PPPOEDiscoveryTag*) (p->pppoedh + PPPOE_DISCOVERY_HEADER_MIN_LEN); - uint32_t pppoe_length = SCNtohs(p->pppoedh->pppoe_length); uint32_t packet_length = len - PPPOE_DISCOVERY_HEADER_MIN_LEN ; @@ -97,29 +92,29 @@ int DecodePPPOEDiscovery(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, return TM_ECODE_OK; } - while (pppoedt < (PPPOEDiscoveryTag*) (pkt + (len - sizeof(PPPOEDiscoveryTag))) && pppoe_length >=4 && packet_length >=4) - { #ifdef DEBUG + /* parse any tags we have in the packet */ + + uint32_t tag_length = 0; + const uint8_t *pkt_pppoedt = pkt + PPPOE_DISCOVERY_HEADER_MIN_LEN; + + // packet_length >= pppoe_length so we have enough data + while (pppoe_length >= sizeof(PPPOEDiscoveryTag)) { + PPPOEDiscoveryTag *pppoedt = (PPPOEDiscoveryTag *)pkt_pppoedt; uint16_t tag_type = SCNtohs(pppoedt->pppoe_tag_type); -#endif + // upgrade to u32 to avoid u16 overflow tag_length = SCNtohs(pppoedt->pppoe_tag_length); SCLogDebug ("PPPoE Tag type %x, length %"PRIu32, tag_type, tag_length); if (pppoe_length >= (4 + tag_length)) { pppoe_length -= (4 + tag_length); + pkt_pppoedt = pkt_pppoedt + (4 + tag_length); } else { pppoe_length = 0; // don't want an underflow } - - if (packet_length >= 4 + tag_length) { - packet_length -= (4 + tag_length); - } else { - packet_length = 0; // don't want an underflow - } - - pppoedt = pppoedt + (4 + tag_length); } +#endif return TM_ECODE_OK; } diff --git a/src/detect-engine-address.c b/src/detect-engine-address.c index ac10e14..191e8f5 100644 --- a/src/detect-engine-address.c +++ b/src/detect-engine-address.c @@ -1362,23 +1362,28 @@ void DetectAddressMapFree(DetectEngineCtx *de_ctx) return; } -static int DetectAddressMapAdd(DetectEngineCtx *de_ctx, const char *string, - DetectAddressHead *address, bool contains_negation) +static bool DetectAddressMapAdd(DetectEngineCtx *de_ctx, const char *string, + DetectAddressHead *address, bool contains_negation) { DetectAddressMap *map = SCCalloc(1, sizeof(*map)); if (map == NULL) - return -1; + return false; map->string = SCStrdup(string); if (map->string == NULL) { SCFree(map); - return -1; + return false; } map->address = address; map->contains_negation = contains_negation; - BUG_ON(HashListTableAdd(de_ctx->address_table, (void *)map, 0) != 0); - return 0; + if (HashListTableAdd(de_ctx->address_table, map, 0) != 0) { + SCFree(map->string); + SCFree(map); + return false; + } + + return true; } static const DetectAddressMap *DetectAddressMapLookup(DetectEngineCtx *de_ctx, @@ -1471,8 +1476,11 @@ const DetectAddressHead *DetectParseAddress(DetectEngineCtx *de_ctx, *contains_negation = false; } - DetectAddressMapAdd((DetectEngineCtx *)de_ctx, string, head, - *contains_negation); + if (!DetectAddressMapAdd((DetectEngineCtx *)de_ctx, string, head, *contains_negation)) { + DetectAddressHeadFree(head); + return NULL; + } + return head; } diff --git a/src/detect-engine-iponly.c b/src/detect-engine-iponly.c index 03b4649..b163277 100644 --- a/src/detect-engine-iponly.c +++ b/src/detect-engine-iponly.c @@ -82,16 +82,78 @@ static IPOnlyCIDRItem *IPOnlyCIDRItemNew(void) SCReturnPtr(item, "IPOnlyCIDRItem"); } -static uint8_t IPOnlyCIDRItemCompare(IPOnlyCIDRItem *head, - IPOnlyCIDRItem *item) +/** + * \brief Compares two list items + * + * \retval An integer less than, equal to, or greater than zero if lhs is + * considered to be respectively less than, equal to, or greater than + * rhs. + */ +static int IPOnlyCIDRItemCompareReal(const IPOnlyCIDRItem *lhs, const IPOnlyCIDRItem *rhs) { - uint8_t i = 0; - for (; i < head->netmask / 32 || i < 1; i++) { - if (item->ip[i] < head->ip[i]) - //if (*(uint8_t *)(item->ip + i) < *(uint8_t *)(head->ip + i)) - return 1; + if (lhs->netmask == rhs->netmask) { + uint8_t i = 0; + for (; i < lhs->netmask / 32 || i < 1; i++) { + if (lhs->ip[i] < rhs->ip[i]) + return -1; + if (lhs->ip[i] > rhs->ip[i]) + return 1; + } + return 0; } - return 0; + + return lhs->netmask < rhs->netmask ? -1 : 1; +} + +static int IPOnlyCIDRItemCompare(const void *lhsv, const void *rhsv) +{ + const IPOnlyCIDRItem *lhs = *(const IPOnlyCIDRItem **)lhsv; + const IPOnlyCIDRItem *rhs = *(const IPOnlyCIDRItem **)rhsv; + + return IPOnlyCIDRItemCompareReal(lhs, rhs); +} + +static void IPOnlyCIDRListQSort(IPOnlyCIDRItem **head) +{ + if (unlikely(head == NULL || *head == NULL)) + return; + + // First count the number of elements in the list + size_t len = 0; + IPOnlyCIDRItem *curr = *head; + + while (curr) { + curr = curr->next; + len++; + } + + // Place a pointer to the list item in an array for sorting + IPOnlyCIDRItem **tmp = SCMalloc(len * sizeof(IPOnlyCIDRItem *)); + + if (unlikely(tmp == NULL)) { + SCLogError("Failed to allocate enough memory to sort IP-only CIDR items."); + return; + } + + curr = *head; + for (size_t i = 0; i < len; i++) { + tmp[i] = curr; + curr = curr->next; + } + + // Perform the sort using the qsort algorithm + qsort(tmp, len, sizeof(IPOnlyCIDRItem *), IPOnlyCIDRItemCompare); + + // Update the links to the next element + *head = tmp[0]; + + for (size_t i = 0; i + 1 < len; i++) { + tmp[i]->next = tmp[i + 1]; + } + + tmp[len - 1]->next = NULL; + + SCFree(tmp); } //declaration for using it already @@ -349,11 +411,9 @@ error: return -1; } - /** * \brief This function insert a IPOnlyCIDRItem - * to a list of IPOnlyCIDRItems sorted by netmask - * ascending + * to a list of IPOnlyCIDRItems * \param head Pointer to the head of IPOnlyCIDRItems list * \param item Pointer to the item to insert in the list * @@ -362,37 +422,12 @@ error: static IPOnlyCIDRItem *IPOnlyCIDRItemInsertReal(IPOnlyCIDRItem *head, IPOnlyCIDRItem *item) { - IPOnlyCIDRItem *it, *prev = NULL; - if (item == NULL) return head; - /* Compare with the head */ - if (item->netmask < head->netmask || (item->netmask == head->netmask && IPOnlyCIDRItemCompare(head, item))) { - item->next = head; - return item; - } - - if (item->netmask == head->netmask && !IPOnlyCIDRItemCompare(head, item)) { - item->next = head->next; - head->next = item; - return head; - } - - for (prev = it = head; - it != NULL && it->netmask < item->netmask; - it = it->next) - prev = it; - - if (it == NULL) { - prev->next = item; - item->next = NULL; - } else { - item->next = it; - prev->next = item; - } - - return head; + /* Always insert item as head */ + item->next = head; + return item; } /** @@ -1112,6 +1147,9 @@ void IPOnlyPrepare(DetectEngineCtx *de_ctx) IPOnlyCIDRListPrint((de_ctx->io_ctx).ip_dst); */ + IPOnlyCIDRListQSort(&(de_ctx->io_ctx).ip_src); + IPOnlyCIDRListQSort(&(de_ctx->io_ctx).ip_dst); + IPOnlyCIDRItem *src, *dst; SCRadixNode *node = NULL; @@ -1729,64 +1767,124 @@ end: static int IPOnlyTestSig04 (void) { int result = 1; - IPOnlyCIDRItem *head = NULL; - IPOnlyCIDRItem *new; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + // Test a linked list of size 0, 1, 2, ..., 5 + for (int size = 0; size < 6; size++) { + IPOnlyCIDRItem *new = NULL; - head = IPOnlyCIDRItemInsert(head, new); + if (size > 0) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 3; - new = IPOnlyCIDRItemNew(); - new->netmask= 11; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 1) { + new = IPOnlyCIDRItemNew(); + new->netmask = 11; - new = IPOnlyCIDRItemNew(); - new->netmask= 9; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 2) { + new = IPOnlyCIDRItemNew(); + new->netmask = 9; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 3) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 1; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 4) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 2; - IPOnlyCIDRListPrint(head); - new = head; - if (new->netmask != 9) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 11) { - result = 0; - goto end; + head = IPOnlyCIDRItemInsert(head, new); + } + + IPOnlyCIDRListPrint(head); + + IPOnlyCIDRListQSort(&head); + + if (size == 0) { + if (head != NULL) { + result = 0; + goto end; + } + } + + /** + * Validate the following list entries for each size + * 1 - 10 + * 2 - 10<3> 11 + * 3 - 9 10<3> 11 + * 4 - 9 10<1> 10<3> 11 + * 5 - 9 10<1> 10<2> 10<3> 11 + */ + new = head; + if (size >= 3) { + if (new->netmask != 9) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 4) { + if (new->netmask != 10 || new->ip[0] != 1) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 5) { + if (new->netmask != 10 || new->ip[0] != 2) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 1) { + if (new->netmask != 10 || new->ip[0] != 3) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 2) { + if (new->netmask != 11) { + result = 0; + goto end; + } + new = new->next; + } + + if (new != NULL) { + result = 0; + goto end; + } + + IPOnlyCIDRListFree(head); + head = NULL; } end: - IPOnlyCIDRListFree(head); + if (head) { + IPOnlyCIDRListFree(head); + head = NULL; + } return result; } diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index e41f277..0cdb453 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -456,6 +456,12 @@ int DetectLoadersSync(void) done = true; } SCMutexUnlock(&loader->m); + if (!done) { + /* nudge thread in case it's sleeping */ + SCCtrlMutexLock(loader->tv->ctrl_mutex); + pthread_cond_broadcast(loader->tv->ctrl_cond); + SCCtrlMutexUnlock(loader->tv->ctrl_mutex); + } } SCMutexLock(&loader->m); if (loader->result != 0) { @@ -511,7 +517,9 @@ static void TmThreadWakeupDetectLoaderThreads(void) while (tv != NULL) { if (strncmp(tv->name,"DL#",3) == 0) { BUG_ON(tv->ctrl_cond == NULL); + SCCtrlMutexLock(tv->ctrl_mutex); pthread_cond_broadcast(tv->ctrl_cond); + SCCtrlMutexUnlock(tv->ctrl_mutex); } tv = tv->next; } @@ -555,6 +563,9 @@ static TmEcode DetectLoaderThreadInit(ThreadVars *t, const void *initdata, void /* pass thread data back to caller */ *data = ftd; + DetectLoaderControl *loader = &loaders[ftd->instance]; + loader->tv = t; + return TM_ECODE_OK; } diff --git a/src/detect-engine-loader.h b/src/detect-engine-loader.h index 7ffb8c8..f43ff9a 100644 --- a/src/detect-engine-loader.h +++ b/src/detect-engine-loader.h @@ -43,9 +43,14 @@ typedef struct DetectLoaderTask_ { typedef struct DetectLoaderControl_ { int id; - int result; /* 0 for ok, error otherwise */ - SCMutex m; - TAILQ_HEAD(, DetectLoaderTask_) task_list; + ThreadVars *tv; /**< loader threads threadvars - for waking them up */ + + /** struct to group members and mutex */ + struct { + SCMutex m; /**< mutex protects result and task_list */ + int result; /**< 0 for ok, error otherwise */ + TAILQ_HEAD(, DetectLoaderTask_) task_list; + }; } DetectLoaderControl; int DetectLoaderQueueTask(int loader_id, LoaderFunc Func, void *func_ctx, LoaderFreeFunc FreeFunc); diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index f091a3d..ede3e59 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -772,18 +772,12 @@ int SignatureHasPacketContent(const Signature *s) { SCEnter(); - if (s == NULL) { - SCReturnInt(0); - } - if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) { SCReturnInt(1); } - if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) || - (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL)) - { - SCLogDebug("no mpm"); + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { + SCLogDebug("no PMATCH"); SCReturnInt(0); } @@ -808,18 +802,12 @@ int SignatureHasStreamContent(const Signature *s) { SCEnter(); - if (s == NULL) { - SCReturnInt(0); - } - if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) { SCReturnInt(0); } - if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) || - (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL)) - { - SCLogDebug("no mpm"); + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { + SCLogDebug("no PMATCH"); SCReturnInt(0); } diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 67af1c1..c75a5d0 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -48,6 +48,7 @@ #include "util-error.h" #include "util-debug.h" +#include "util-validate.h" #include "util-cidr.h" #include "util-unittest.h" #include "util-unittest-helper.h" @@ -637,6 +638,8 @@ void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh) continue; if (SignatureIsFilestoring(s)) { + // should be insured by caller that we do not overflow + DEBUG_VALIDATE_BUG_ON(sgh->filestore_cnt == UINT16_MAX); sgh->filestore_cnt++; } } diff --git a/src/detect-filestore.c b/src/detect-filestore.c index c53a93d..c905f9b 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -118,7 +118,8 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto switch (filestore->direction) { case FILESTORE_DIR_DEFAULT: rule_dir = 1; - break; + // will use both sides if scope is not default + // fallthrough case FILESTORE_DIR_BOTH: toserver_dir = 1; toclient_dir = 1; @@ -160,16 +161,28 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv); DEBUG_VALIDATE_BUG_ON(txd == NULL); if (txd != NULL) { - txd->file_flags |= FLOWFILE_STORE; + if (toclient_dir) { + txd->file_flags |= FLOWFILE_STORE_TC; + } + if (toserver_dir) { + txd->file_flags |= FLOWFILE_STORE_TS; + } } } } else if (this_flow) { /* set in flow and AppLayerStateData */ - f->file_flags |= FLOWFILE_STORE; - AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate); - if (sd != NULL) { - sd->file_flags |= FLOWFILE_STORE; + if (toclient_dir) { + f->file_flags |= FLOWFILE_STORE_TC; + if (sd != NULL) { + sd->file_flags |= FLOWFILE_STORE_TC; + } + } + if (toserver_dir) { + f->file_flags |= FLOWFILE_STORE_TS; + if (sd != NULL) { + sd->file_flags |= FLOWFILE_STORE_TS; + } } } else { FileStoreFileById(fc, file_id); @@ -333,6 +346,11 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch static bool warn_not_configured = false; static uint32_t de_version = 0; + if (de_ctx->filestore_cnt == UINT16_MAX) { + SCLogError("Cannot have more than 65535 filestore signatures"); + return -1; + } + /* Check on first-time loads (includes following a reload) */ if (!warn_not_configured || (de_ctx->version != de_version)) { if (de_version != de_ctx->version) { @@ -476,6 +494,7 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_POSTMATCH); s->flags |= SIG_FLAG_FILESTORE; + de_ctx->filestore_cnt++; if (match) pcre2_match_data_free(match); diff --git a/src/detect-http-header.c b/src/detect-http-header.c index 2803d05..cd36ea5 100644 --- a/src/detect-http-header.c +++ b/src/detect-http-header.c @@ -600,6 +600,13 @@ typedef struct HttpMultiBufHeaderThreadData { static void *HttpMultiBufHeaderThreadDataInit(void *data) { HttpMultiBufHeaderThreadData *td = SCCalloc(1, sizeof(*td)); + + /* This return value check to satisfy our Cocci malloc checks. */ + if (td == NULL) { + SCLogError("failed to allocate %" PRIuMAX " bytes: %s", (uintmax_t)sizeof(*td), + strerror(errno)); + return NULL; + } return td; } @@ -668,10 +675,11 @@ static InspectionBuffer *GetHttp1HeaderData(DetectEngineThreadCtx *det_ctx, cons size_t size = size1 + size2 + 2; if (hdr_td->items[i].len < size) { // Use realloc, as this pointer is not freed until HttpMultiBufHeaderThreadDataFree - hdr_td->items[i].buffer = SCRealloc(hdr_td->items[i].buffer, size); - if (unlikely(hdr_td->items[i].buffer == NULL)) { + void *tmp = SCRealloc(hdr_td->items[i].buffer, size); + if (unlikely(tmp == NULL)) { return NULL; } + hdr_td->items[i].buffer = tmp; } memcpy(hdr_td->items[i].buffer, bstr_ptr(h->name), size1); hdr_td->items[i].buffer[size1] = ':'; diff --git a/src/detect-parse.c b/src/detect-parse.c index b5e214d..c3232b9 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1540,6 +1540,7 @@ Signature *SigAlloc (void) sig->init_data->buffers = SCCalloc(8, sizeof(SignatureInitDataBuffer)); if (sig->init_data->buffers == NULL) { + SCFree(sig->init_data); SCFree(sig); return NULL; } diff --git a/src/detect-tls-certs.c b/src/detect-tls-certs.c index a020437..f233779 100644 --- a/src/detect-tls-certs.c +++ b/src/detect-tls-certs.c @@ -70,6 +70,7 @@ static int g_tls_certs_buffer_id = 0; struct TlsCertsGetDataArgs { uint32_t local_id; /**< used as index into thread inspect array */ SSLCertsChain *cert; + const uint8_t flags; }; typedef struct PrefilterMpmTlsCerts { @@ -150,7 +151,7 @@ static InspectionBuffer *TlsCertsGetData(DetectEngineThreadCtx *det_ctx, const SSLState *ssl_state = (SSLState *)f->alstate; const SSLStateConnp *connp; - if (f->flags & STREAM_TOSERVER) { + if (cbdata->flags & STREAM_TOSERVER) { connp = &ssl_state->client_connp; } else { connp = &ssl_state->server_connp; @@ -185,7 +186,7 @@ static uint8_t DetectEngineInspectTlsCerts(DetectEngineCtx *de_ctx, DetectEngine transforms = engine->v2.transforms; } - struct TlsCertsGetDataArgs cbdata = { 0, NULL }; + struct TlsCertsGetDataArgs cbdata = { .local_id = 0, .cert = NULL, .flags = flags }; while (1) { @@ -222,7 +223,7 @@ static void PrefilterTxTlsCerts(DetectEngineThreadCtx *det_ctx, const void *pect const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - struct TlsCertsGetDataArgs cbdata = { 0, NULL }; + struct TlsCertsGetDataArgs cbdata = { .local_id = 0, .cert = NULL, .flags = flags }; while (1) { diff --git a/src/detect.h b/src/detect.h index 0186545..587a29c 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1039,6 +1039,9 @@ typedef struct DetectEngineCtx_ { /* Track rule requirements for reporting after loading rules. */ SCDetectRequiresStatus *requirements; + + /* number of signatures using filestore, limited as u16 */ + uint16_t filestore_cnt; } DetectEngineCtx; /* Engine groups profiles (low, medium, high, custom) */ diff --git a/src/flow-timeout.c b/src/flow-timeout.c index 6a9b707..e5d2794 100644 --- a/src/flow-timeout.c +++ b/src/flow-timeout.c @@ -213,7 +213,7 @@ static inline Packet *FlowForceReassemblyPseudoPacketSetup(Packet *p, } p->tcph->th_offx2 = 0x50; - p->tcph->th_flags |= TH_ACK; + p->tcph->th_flags = 0; p->tcph->th_win = 10; p->tcph->th_urp = 0; diff --git a/src/flow-worker.c b/src/flow-worker.c index a20e053..32fbe09 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -391,8 +391,16 @@ static inline void FlowWorkerStreamTCPUpdate(ThreadVars *tv, FlowWorkerThreadDat StreamTcp(tv, p, fw->stream_thread, &fw->pq); FLOWWORKER_PROFILING_END(p, PROFILE_FLOWWORKER_STREAM); - if (FlowChangeProto(p->flow)) { + // this is the first packet that sets no payload inspection + bool setting_nopayload = + p->flow->alparser && + AppLayerParserStateIssetFlag(p->flow->alparser, APP_LAYER_PARSER_NO_INSPECTION) && + !(p->flags & PKT_NOPAYLOAD_INSPECTION); + if (FlowChangeProto(p->flow) || setting_nopayload) { StreamTcpDetectLogFlush(tv, fw->stream_thread, p->flow, p, &fw->pq); + if (setting_nopayload) { + FlowSetNoPayloadInspectionFlag(p->flow); + } AppLayerParserStateSetFlag(p->flow->alparser, APP_LAYER_PARSER_EOF_TS); AppLayerParserStateSetFlag(p->flow->alparser, APP_LAYER_PARSER_EOF_TC); } @@ -430,6 +438,10 @@ static inline void FlowWorkerStreamTCPUpdate(ThreadVars *tv, FlowWorkerThreadDat TmqhOutputPacketpool(tv, x); } } + if (FlowChangeProto(p->flow) && p->flow->flags & FLOW_ACTION_DROP) { + // in case f->flags & FLOW_ACTION_DROP was set by one of the dequeued packets + PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_FLOW_DROP); + } } static void FlowWorkerFlowTimeout(ThreadVars *tv, Packet *p, FlowWorkerThreadData *fw, @@ -142,8 +142,9 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOWFILE_NO_SIZE_TS BIT_U16(10) #define FLOWFILE_NO_SIZE_TC BIT_U16(11) -/** store all files in the flow */ -#define FLOWFILE_STORE BIT_U16(12) +/** store files in the flow */ +#define FLOWFILE_STORE_TS BIT_U16(12) +#define FLOWFILE_STORE_TC BIT_U16(13) #define FLOWFILE_NONE_TS (FLOWFILE_NO_MAGIC_TS | \ FLOWFILE_NO_STORE_TS | \ diff --git a/src/output-filestore.c b/src/output-filestore.c index dcf4c1a..d23560c 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -194,12 +194,8 @@ static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len); - char base_filename[PATH_MAX] = ""; - snprintf(base_filename, sizeof(base_filename), "%s/file.%u", - ctx->tmpdir, ff->file_store_id); - snprintf(filename, sizeof(filename), "%s", base_filename); - if (flags & OUTPUT_FILEDATA_FLAG_OPEN) { + snprintf(filename, sizeof(filename), "%s/file.%u", ctx->tmpdir, ff->file_store_id); file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644); if (file_fd == -1) { @@ -220,6 +216,7 @@ static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet /* we can get called with a NULL ffd when we need to close */ } else if (data != NULL) { if (ff->fd == -1) { + snprintf(filename, sizeof(filename), "%s/file.%u", ctx->tmpdir, ff->file_store_id); file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY); if (file_fd == -1) { StatsIncr(tv, aft->fs_error_counter); @@ -235,6 +232,7 @@ static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet if (file_fd != -1) { ssize_t r = write(file_fd, (const void *)data, (size_t)data_len); if (r == -1) { + snprintf(filename, sizeof(filename), "%s/file.%u", ctx->tmpdir, ff->file_store_id); StatsIncr(tv, aft->fs_error_counter); WARN_ONCE(WOT_WRITE, "Filestore (v2) failed to write to %s: %s", filename, strerror(errno)); diff --git a/src/output-json-stats.c b/src/output-json-stats.c index 718298e..33f98af 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -36,6 +36,7 @@ #include "util-print.h" #include "util-time.h" #include "util-unittest.h" +#include "util-validate.h" #include "util-debug.h" #include "output.h" @@ -265,20 +266,30 @@ json_t *StatsToJSON(const StatsTable *st, uint8_t flags) for (x = 0; x < st->ntstats; x++) { uint32_t offset = x * st->nstats; + // Stats for for this thread. + json_t *thread = json_object(); + if (unlikely(thread == NULL)) { + json_decref(js_stats); + json_decref(threads); + return NULL; + } + /* for each counter */ for (u = offset; u < (offset + st->nstats); u++) { if (st->tstats[u].name == NULL) continue; + // Seems this holds, but assert in debug builds. + DEBUG_VALIDATE_BUG_ON( + strcmp(st->tstats[offset].tm_name, st->tstats[u].tm_name) != 0); + json_t *js_type = NULL; const char *stat_name = st->tstats[u].short_name; if (st->tstats[u].short_name == NULL) { stat_name = st->tstats[u].name; js_type = threads; } else { - char str[256]; - snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name); - js_type = OutputStats2Json(threads, str); + js_type = OutputStats2Json(thread, st->tstats[u].name); } if (js_type != NULL) { @@ -292,6 +303,7 @@ json_t *StatsToJSON(const StatsTable *st, uint8_t flags) } } } + json_object_set_new(threads, st->tstats[offset].tm_name, thread); } json_object_set_new(js_stats, "threads", threads); } @@ -471,3 +483,7 @@ void JsonStatsLogRegister(void) { "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger, JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL); } + +#ifdef UNITTESTS +#include "tests/output-json-stats.c" +#endif diff --git a/src/output-json-stats.h b/src/output-json-stats.h index 9b96d50..b569e30 100644 --- a/src/output-json-stats.h +++ b/src/output-json-stats.h @@ -35,4 +35,6 @@ TmEcode OutputEngineStatsReloadTime(json_t **jdata); TmEcode OutputEngineStatsRuleset(json_t **jdata); void JsonStatsLogRegister(void); +void OutputJsonStatsRegisterTests(void); + #endif /* __OUTPUT_JSON_COUNTERS_H__ */ diff --git a/src/output-tx.c b/src/output-tx.c index 18a34e7..042b424 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -339,7 +339,9 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) DEBUG_VALIDATE_BUG_ON(thread_data == NULL); if (p->flow == NULL) return TM_ECODE_OK; - if (!((PKT_IS_PSEUDOPKT(p)) || p->flow->flags & (FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATED))) { + if (!PKT_IS_PSEUDOPKT(p) && p->app_update_direction == 0 && + ((PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATED) == 0) || + (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATED) == 0))) { SCLogDebug("not pseudo, no app update: skip"); return TM_ECODE_OK; } diff --git a/src/runmode-dpdk.c b/src/runmode-dpdk.c index 2cdf5cb..1a240aa 100644 --- a/src/runmode-dpdk.c +++ b/src/runmode-dpdk.c @@ -464,6 +464,9 @@ static int ConfigSetMempoolSize(DPDKIfaceConfig *iconf, intmax_t entry_int) if (entry_int <= 0) { SCLogError("%s: positive memory pool size is required", iconf->iface); SCReturnInt(-ERANGE); + } else if (entry_int > UINT32_MAX) { + SCLogError("%s: memory pool size cannot exceed %" PRIu32, iconf->iface, UINT32_MAX); + SCReturnInt(-ERANGE); } iconf->mempool_size = entry_int; @@ -484,7 +487,7 @@ static int ConfigSetMempoolCacheSize(DPDKIfaceConfig *iconf, const char *entry_s SCReturnInt(-EINVAL); } - uint32_t max_cache_size = MAX(RTE_MEMPOOL_CACHE_MAX_SIZE, iconf->mempool_size / 1.5); + uint32_t max_cache_size = MIN(RTE_MEMPOOL_CACHE_MAX_SIZE, iconf->mempool_size / 1.5); iconf->mempool_cache_size = GreatestDivisorUpTo(iconf->mempool_size, max_cache_size); SCReturnInt(0); } @@ -510,6 +513,9 @@ static int ConfigSetRxDescriptors(DPDKIfaceConfig *iconf, intmax_t entry_int) if (entry_int <= 0) { SCLogError("%s: positive number of RX descriptors is required", iconf->iface); SCReturnInt(-ERANGE); + } else if (entry_int > UINT16_MAX) { + SCLogError("%s: number of RX descriptors cannot exceed %" PRIu16, iconf->iface, UINT16_MAX); + SCReturnInt(-ERANGE); } iconf->nb_rx_desc = entry_int; @@ -522,6 +528,9 @@ static int ConfigSetTxDescriptors(DPDKIfaceConfig *iconf, intmax_t entry_int) if (entry_int <= 0) { SCLogError("%s: positive number of TX descriptors is required", iconf->iface); SCReturnInt(-ERANGE); + } else if (entry_int > UINT16_MAX) { + SCLogError("%s: number of TX descriptors cannot exceed %" PRIu16, iconf->iface, UINT16_MAX); + SCReturnInt(-ERANGE); } iconf->nb_tx_desc = entry_int; @@ -1424,12 +1433,17 @@ static int DeviceConfigure(DPDKIfaceConfig *iconf) if (retval < 0) return retval; + uint16_t tmp_nb_rx_desc = iconf->nb_rx_desc; + uint16_t tmp_nb_tx_desc = iconf->nb_tx_desc; retval = rte_eth_dev_adjust_nb_rx_tx_desc( iconf->port_id, &iconf->nb_rx_desc, &iconf->nb_tx_desc); if (retval != 0) { SCLogError("%s: failed to adjust device queue descriptors (port %u, err %d)", iconf->iface, iconf->port_id, retval); SCReturnInt(retval); + } else if (tmp_nb_rx_desc != iconf->nb_rx_desc || tmp_nb_tx_desc != iconf->nb_tx_desc) { + SCLogWarning("%s: device queue descriptors adjusted (RX: from %u to %u, TX: from %u to %u)", + iconf->iface, tmp_nb_rx_desc, iconf->nb_rx_desc, tmp_nb_tx_desc, iconf->nb_tx_desc); } retval = iconf->flags & DPDK_MULTICAST ? rte_eth_allmulticast_enable(iconf->port_id) diff --git a/src/runmode-napatech.c b/src/runmode-napatech.c index cb8f560..fe02124 100644 --- a/src/runmode-napatech.c +++ b/src/runmode-napatech.c @@ -200,7 +200,12 @@ static void *NapatechConfigParser(const char *device) if (ConfGetInt("napatech.hba", &conf->hba) == 0) { conf->hba = -1; } else { - SCLogWarning("Napatech Host Buffer Allocation (hba) will be deprecated in Suricata v7.0."); + static bool warn_once = false; + if (!warn_once) { + SCLogWarning( + "Napatech Host Buffer Allowance (hba) will be deprecated in Suricata v8.0."); + warn_once = true; + } } return (void *) conf; } diff --git a/src/runmode-netmap.c b/src/runmode-netmap.c index 927dc71..e207cf0 100644 --- a/src/runmode-netmap.c +++ b/src/runmode-netmap.c @@ -344,7 +344,9 @@ static void *ParseNetmapConfig(const char *iface_name) } } - int ring_count = NetmapGetRSSCount(aconf->iface_name); + int ring_count = 0; + if (aconf->in.real) + ring_count = NetmapGetRSSCount(aconf->iface_name); if (strlen(aconf->iface_name) > 0 && (aconf->iface_name[strlen(aconf->iface_name) - 1] == '^' || aconf->iface_name[strlen(aconf->iface_name) - 1] == '*')) { diff --git a/src/runmode-pfring.c b/src/runmode-pfring.c index b0af83b..7f1f74f 100644 --- a/src/runmode-pfring.c +++ b/src/runmode-pfring.c @@ -200,6 +200,7 @@ static void *ParsePfringConfig(const char *iface) cluster_type default_ctype = CLUSTER_FLOW; int getctype = 0; int bool_val; + const char *active_runmode = RunmodeGetActive(); if (unlikely(pfconf == NULL)) { return NULL; @@ -244,7 +245,9 @@ static void *ParsePfringConfig(const char *iface) if_default = NULL; } - if (ConfGetChildValueWithDefault(if_root, if_default, "threads", &threadsstr) != 1) { + if (active_runmode && !strcmp("single", active_runmode)) { + pfconf->threads = 1; + } else if (ConfGetChildValueWithDefault(if_root, if_default, "threads", &threadsstr) != 1) { pfconf->threads = 1; } else if (threadsstr != NULL) { if (strcmp(threadsstr, "auto") == 0) { diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index 1150bad..8ce0244 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -114,6 +114,8 @@ #include "decode-vntag.h" #include "decode-vxlan.h" +#include "output-json-stats.h" + #ifdef OS_WIN32 #include "win32-syscall.h" #endif @@ -215,6 +217,7 @@ static void RegisterUnittests(void) #endif SCProtoNameRegisterTests(); UtilCIDRTests(); + OutputJsonStatsRegisterTests(); } #endif diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index e695cb8..8b26990 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -545,7 +545,7 @@ static TmEcode UnixSocketPcapFilesCheck(void *data) if (cfile->tenant_id > 0) { char tstr[16]; - snprintf(tstr, sizeof(tstr), "%d", cfile->tenant_id); + snprintf(tstr, sizeof(tstr), "%u", cfile->tenant_id); if (ConfSetFinal("pcap-file.tenant-id", tstr) != 1) { SCLogError("Can not set working tenant-id to '%s'", tstr); PcapFilesFree(cfile); @@ -1038,7 +1038,7 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data) /* setup the yaml in this loop so that it's not done by the loader * threads. ConfYamlLoadFileWithPrefix is not thread safe. */ char prefix[64]; - snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id); + snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id); if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) { SCLogError("failed to load yaml %s", filename); json_object_set_new(answer, "message", json_string("failed to load yaml")); @@ -1187,7 +1187,7 @@ TmEcode UnixSocketUnregisterTenant(json_t *cmd, json_t* answer, void *data) /* 2 remove it from the system */ char prefix[64]; - snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id); + snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id); DetectEngineCtx *de_ctx = DetectEngineGetByTenantId(tenant_id); if (de_ctx == NULL) { diff --git a/src/source-dpdk.c b/src/source-dpdk.c index 54503e2..cf26af5 100644 --- a/src/source-dpdk.c +++ b/src/source-dpdk.c @@ -564,7 +564,7 @@ static TmEcode ReceiveDPDKThreadInit(ThreadVars *tv, const void *initdata, void if (inconsistent_numa_cnt > 0 && ptv->port_socket_id != SOCKET_ID_ANY) { SCLogWarning("%s: NIC is on NUMA %d, %u threads on different NUMA node(s)", dpdk_config->iface, ptv->port_socket_id, inconsistent_numa_cnt); - } else if (ptv->port_socket_id == SOCKET_ID_ANY) { + } else if (ptv->port_socket_id == SOCKET_ID_ANY && rte_socket_count() > 1) { SCLogNotice( "%s: unable to determine NIC's NUMA node, degraded performance can be expected", dpdk_config->iface); diff --git a/src/source-netmap.c b/src/source-netmap.c index 0b04b41..8e409ea 100644 --- a/src/source-netmap.c +++ b/src/source-netmap.c @@ -453,6 +453,7 @@ retry: } } + SCMutexUnlock(&netmap_devlist_lock); NetmapCloseAll(); FatalError("opening devname %s failed: %s", devname, strerror(errno)); } diff --git a/src/source-pfring.c b/src/source-pfring.c index 96da94e..10eac2f 100644 --- a/src/source-pfring.c +++ b/src/source-pfring.c @@ -430,6 +430,7 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot) } } else if (unlikely(r == 0)) { if (suricata_ctl_flags & SURICATA_STOP) { + TmqhOutputPacketpool(ptv->tv, p); SCReturnInt(TM_ECODE_OK); } @@ -701,6 +702,7 @@ TmEcode ReceivePfringThreadDeinit(ThreadVars *tv, void *data) } pfring_close(ptv->pd); + SCFree(ptv); return TM_ECODE_OK; } diff --git a/src/suricata.c b/src/suricata.c index d0e1049..1c5ac7c 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -881,9 +881,6 @@ int g_ut_covered; void RegisterAllModules(void) { - // zero all module storage - memset(tmm_modules, 0, TMM_SIZE * sizeof(TmModule)); - /* commanders */ TmModuleUnixManagerRegister(); /* managers */ @@ -2672,6 +2669,10 @@ int PostConfLoadedSetup(SCInstance *suri) MacSetRegisterFlowStorage(); +#ifdef HAVE_PLUGINS + SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); +#endif + LiveDeviceFinalize(); // must be after EBPF extension registration RunModeEngineIsIPS( @@ -2743,9 +2744,6 @@ int PostConfLoadedSetup(SCInstance *suri) FeatureTrackingRegister(); /* must occur prior to output mod registration */ RegisterAllModules(); -#ifdef HAVE_PLUGINS - SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); -#endif AppLayerHtpNeedFileInspection(); StorageFinalize(); @@ -2869,6 +2867,10 @@ int InitGlobal(void) ConfInit(); VarNameStoreInit(); + + // zero all module storage + memset(tmm_modules, 0, TMM_SIZE * sizeof(TmModule)); + return 0; } @@ -2971,7 +2973,10 @@ int SuricataMain(int argc, char **argv) goto out; } - SystemHugepageSnapshot *prerun_snap = SystemHugepageSnapshotCreate(); + SystemHugepageSnapshot *prerun_snap = NULL; + if (run_mode == RUNMODE_DPDK) + prerun_snap = SystemHugepageSnapshotCreate(); + SCSetStartTime(&suricata); RunModeDispatch(suricata.run_mode, suricata.runmode_custom_mode, suricata.capture_plugin_name, suricata.capture_plugin_args); @@ -3029,13 +3034,12 @@ int SuricataMain(int argc, char **argv) OnNotifyRunning(); PostRunStartedDetectSetup(&suricata); - - SystemHugepageSnapshot *postrun_snap = SystemHugepageSnapshotCreate(); - if (run_mode == RUNMODE_DPDK) // only DPDK uses hpages at the moment + if (run_mode == RUNMODE_DPDK) { // only DPDK uses hpages at the moment + SystemHugepageSnapshot *postrun_snap = SystemHugepageSnapshotCreate(); SystemHugepageEvaluateHugepages(prerun_snap, postrun_snap); - SystemHugepageSnapshotDestroy(prerun_snap); - SystemHugepageSnapshotDestroy(postrun_snap); - + SystemHugepageSnapshotDestroy(prerun_snap); + SystemHugepageSnapshotDestroy(postrun_snap); + } SCPledge(); SuricataMainLoop(&suricata); diff --git a/src/tests/output-json-stats.c b/src/tests/output-json-stats.c new file mode 100644 index 0000000..ac1336e --- /dev/null +++ b/src/tests/output-json-stats.c @@ -0,0 +1,70 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "../suricata-common.h" + +#include "../output-json-stats.h" + +#include "../util-unittest.h" + +static int OutputJsonStatsTest01(void) +{ + StatsRecord global_records[] = { { 0 }, { 0 } }; + StatsRecord thread_records[2]; + thread_records[0].name = "capture.kernel_packets"; + thread_records[0].short_name = "kernel_packets"; + thread_records[0].tm_name = "W#01-bond0.30"; + thread_records[0].value = 42; + thread_records[1].name = "capture.kernel_drops"; + thread_records[1].short_name = "kernel_drops"; + thread_records[1].tm_name = "W#01-bond0.30"; + thread_records[1].value = 4711; + + StatsTable table = { + .nstats = 2, + .stats = &global_records[0], + .ntstats = 1, + .tstats = &thread_records[0], + }; + + json_t *r = StatsToJSON(&table, JSON_STATS_TOTALS | JSON_STATS_THREADS); + if (!r) + return 0; + + // Remove variable content + json_object_del(r, "uptime"); + + char *serialized = json_dumps(r, 0); + + // Cheesy comparison + const char *expected = "{\"threads\": {\"W#01-bond0.30\": {\"capture\": {\"kernel_packets\": " + "42, \"kernel_drops\": 4711}}}}"; + + int cmp_result = strcmp(expected, serialized); + if (cmp_result != 0) + printf("unexpected result\nexpected=%s\ngot=%s\n", expected, serialized); + + free(serialized); + json_decref(r); + + return cmp_result == 0; +} + +void OutputJsonStatsRegisterTests(void) +{ + UtRegisterTest("OutputJsonStatsTest01", OutputJsonStatsTest01); +} diff --git a/src/tm-threads.c b/src/tm-threads.c index b173cb8..1853db6 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1241,13 +1241,17 @@ static int TmThreadKillThread(ThreadVars *tv) } if (tv->inq != NULL) { for (int i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) { + SCMutexLock(&tv->inq->pq->mutex_q); SCCondSignal(&tv->inq->pq->cond_q); + SCMutexUnlock(&tv->inq->pq->mutex_q); } SCLogDebug("signalled tv->inq->id %" PRIu32 "", tv->inq->id); } if (tv->ctrl_cond != NULL ) { + SCCtrlMutexLock(tv->ctrl_mutex); pthread_cond_broadcast(tv->ctrl_cond); + SCCtrlMutexUnlock(tv->ctrl_mutex); } return 0; } @@ -1427,7 +1431,9 @@ again: if (tv->inq != NULL) { for (int i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) { + SCMutexLock(&tv->inq->pq->mutex_q); SCCondSignal(&tv->inq->pq->cond_q); + SCMutexUnlock(&tv->inq->pq->mutex_q); } SCLogDebug("signalled tv->inq->id %" PRIu32 "", tv->inq->id); } @@ -1507,7 +1513,9 @@ again: * THV_KILL flag. */ if (tv->inq != NULL) { for (int i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) { + SCMutexLock(&tv->inq->pq->mutex_q); SCCondSignal(&tv->inq->pq->cond_q); + SCMutexUnlock(&tv->inq->pq->mutex_q); } SCLogDebug("signalled tv->inq->id %" PRIu32 "", tv->inq->id); } @@ -2298,7 +2306,9 @@ void TmThreadsInjectFlowById(Flow *f, const int id) /* wake up listening thread(s) if necessary */ if (tv->inq != NULL) { + SCMutexLock(&tv->inq->pq->mutex_q); SCCondSignal(&tv->inq->pq->cond_q); + SCMutexUnlock(&tv->inq->pq->mutex_q); } else if (tv->break_loop) { TmThreadsCaptureBreakLoop(tv); } diff --git a/src/tmqh-simple.c b/src/tmqh-simple.c index 47faed5..0bfa173 100644 --- a/src/tmqh-simple.c +++ b/src/tmqh-simple.c @@ -76,8 +76,11 @@ void TmqhInputSimpleShutdownHandler(ThreadVars *tv) return; } - for (i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) + for (i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) { + SCMutexLock(&tv->inq->pq->mutex_q); SCCondSignal(&tv->inq->pq->cond_q); + SCMutexUnlock(&tv->inq->pq->mutex_q); + } } void TmqhOutputSimple(ThreadVars *t, Packet *p) diff --git a/src/util-decode-mime.c b/src/util-decode-mime.c index 5e7a8d5..eb67c3d 100644 --- a/src/util-decode-mime.c +++ b/src/util-decode-mime.c @@ -2439,6 +2439,7 @@ MimeDecParseState * MimeDecInitParser(void *data, PushStack(state->stack); if (state->stack->top == NULL) { SCFree(state->stack); + SCFree(state->msg); SCFree(state); return NULL; } diff --git a/src/util-error.c b/src/util-error.c index 01c2f9a..e3195a1 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -47,6 +47,7 @@ const char * SCErrorToString(SCError err) CASE_CODE(SC_EINVAL); CASE_CODE(SC_ELIMIT); CASE_CODE(SC_EEXIST); + CASE_CODE(SC_ENOENT); CASE_CODE (SC_ERR_MAX); } diff --git a/src/util-error.h b/src/util-error.h index eaaf8cb..f1bc80d 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -30,6 +30,7 @@ typedef enum { SC_EINVAL, SC_ELIMIT, SC_EEXIST, + SC_ENOENT, SC_ERR_MAX } SCError; diff --git a/src/util-file.c b/src/util-file.c index 0449a2e..89ef50c 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -235,8 +235,11 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) uint16_t flags = 0; if (direction == STREAM_TOSERVER) { - if ((flow_file_flags & (FLOWFILE_NO_STORE_TS | FLOWFILE_STORE)) == FLOWFILE_NO_STORE_TS) { + if ((flow_file_flags & (FLOWFILE_NO_STORE_TS | FLOWFILE_STORE_TS)) == + FLOWFILE_NO_STORE_TS) { flags |= FILE_NOSTORE; + } else if (flow_file_flags & FLOWFILE_STORE_TS) { + flags |= FILE_STORE; } if (flow_file_flags & FLOWFILE_NO_MAGIC_TS) { @@ -255,8 +258,11 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) flags |= FILE_NOSHA256; } } else { - if ((flow_file_flags & (FLOWFILE_NO_STORE_TC | FLOWFILE_STORE)) == FLOWFILE_NO_STORE_TC) { + if ((flow_file_flags & (FLOWFILE_NO_STORE_TC | FLOWFILE_STORE_TC)) == + FLOWFILE_NO_STORE_TC) { flags |= FILE_NOSTORE; + } else if (flow_file_flags & FLOWFILE_STORE_TC) { + flags |= FILE_STORE; } if (flow_file_flags & FLOWFILE_NO_MAGIC_TC) { @@ -275,9 +281,6 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) flags |= FILE_NOSHA256; } } - if (flow_file_flags & FLOWFILE_STORE) { - flags |= FILE_STORE; - } DEBUG_VALIDATE_BUG_ON((flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE)); SCLogDebug("direction %02x flags %02x", direction, flags); diff --git a/src/util-hugepages.c b/src/util-hugepages.c index 2af74c3..5ad3519 100644 --- a/src/util-hugepages.c +++ b/src/util-hugepages.c @@ -24,6 +24,7 @@ #include "suricata.h" #include "util-debug.h" #include "util-hugepages.h" +#include "util-path.h" static uint16_t SystemHugepageSizesCntPerNodeGet(uint16_t node_index); static uint16_t SystemNodeCountGet(void); @@ -36,18 +37,28 @@ static void SystemHugepageNodeInfoDestroy(NodeInfo *n); static void SystemHugepageNodeInfoDump(NodeInfo *n); static void SystemHugepageSnapshotDump(SystemHugepageSnapshot *s); +typedef enum OSHugepageAction_ { + OS_UNKNOWN, // unknown/unsupported OS + OS_LINUX_SYS_DEVICES, +} OSHugepageAction; + +static OSHugepageAction SystemHugepageDetermineOS(void) +{ + // try Linux + if (SCPathExists("/sys/devices/system/node/")) { + return OS_LINUX_SYS_DEVICES; + } + + return OS_UNKNOWN; +} + static bool SystemHugepageSupported(void) { -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - return true; -#else + if (SystemHugepageDetermineOS() != OS_UNKNOWN) + return true; return false; -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ } -// block of all hugepage-specific internal functions -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - /** * \brief Linux-specific function to detect number of NUMA nodes on the system * \returns number of NUMA nodes, 0 on error @@ -56,16 +67,14 @@ static uint16_t SystemNodeCountGetLinux(void) { char dir_path[] = "/sys/devices/system/node/"; DIR *dir = opendir(dir_path); - if (dir == NULL) { - SCLogError("unable to open %s", dir_path); - return 0; - } + if (dir == NULL) + FatalError("unable to open %s", dir_path); uint16_t count = 0; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { char d_name[] = "node"; - if (entry->d_type == DT_DIR && strncmp(entry->d_name, d_name, strlen(d_name)) == 0) + if (SCIsRegularDirectory(entry) && strncmp(entry->d_name, d_name, strlen(d_name)) == 0) count++; } closedir(dir); @@ -83,7 +92,7 @@ static uint16_t SystemHugepageSizesCntPerNodeGetLinux(uint16_t node_index) snprintf(dir_path, sizeof(dir_path), "/sys/devices/system/node/node%d/hugepages/", node_index); DIR *dir = opendir(dir_path); if (dir == NULL) { - SCLogError("unable to open %s", dir_path); + SCLogInfo("unable to open %s", dir_path); return 0; } @@ -91,7 +100,7 @@ static uint16_t SystemHugepageSizesCntPerNodeGetLinux(uint16_t node_index) struct dirent *entry; while ((entry = readdir(dir)) != NULL) { char d_name[] = "hugepages-"; - if (entry->d_type == DT_DIR && strncmp(entry->d_name, d_name, strlen(d_name)) == 0) + if (SCIsRegularDirectory(entry) && strncmp(entry->d_name, d_name, strlen(d_name)) == 0) count++; } closedir(dir); @@ -111,14 +120,13 @@ static void SystemHugepagePerNodeGetHugepageSizesLinux( char dir_path[256]; snprintf(dir_path, sizeof(dir_path), "/sys/devices/system/node/node%d/hugepages/", node_index); DIR *dir = opendir(dir_path); - if (dir == NULL) { - SCLogError("unable to open %s", dir_path); - return; - } + if (dir == NULL) + FatalError("unable to open %s", dir_path); + uint16_t index = 0; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { - if (entry->d_type == DT_DIR && strncmp(entry->d_name, "hugepages-", 10) == 0) { + if (SCIsRegularDirectory(entry) && strncmp(entry->d_name, "hugepages-", 10) == 0) { sscanf(entry->d_name, "hugepages-%ukB", &(hp_sizes[index])); index++; } @@ -146,11 +154,11 @@ static int16_t SystemHugepagePerNodeGetHugepageInfoLinux( node_index, hp_sizes[i]); FILE *f = fopen(path, "r"); if (!f) { - SCLogError("unable to open %s", path); - return -SC_EEXIST; + SCLogInfo("unable to open %s", path); + return -SC_ENOENT; } if (fscanf(f, "%hu", &hugepages[i].allocated) != 1) { - SCLogError("failed to read the total number of allocated hugepages (%ukB) on node %hu", + SCLogInfo("failed to read the total number of allocated hugepages (%ukB) on node %hu", hp_sizes[i], node_index); fclose(f); return -SC_EINVAL; @@ -162,11 +170,11 @@ static int16_t SystemHugepagePerNodeGetHugepageInfoLinux( node_index, hp_sizes[i]); f = fopen(path, "r"); if (!f) { - SCLogError("unable to open %s", path); - return -SC_EEXIST; + SCLogInfo("unable to open %s", path); + return -SC_ENOENT; } if (fscanf(f, "%hu", &hugepages[i].free) != 1) { - SCLogError("failed to read the total number of free hugepages (%ukB) on node %hu", + SCLogInfo("failed to read the total number of free hugepages (%ukB) on node %hu", hp_sizes[i], node_index); fclose(f); return -SC_EINVAL; @@ -177,8 +185,6 @@ static int16_t SystemHugepagePerNodeGetHugepageInfoLinux( return 0; } -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ - /** * \brief The function gathers information about hugepages on a given node * \param[in] node_index index of the NUMA node @@ -189,8 +195,8 @@ static int16_t SystemHugepagePerNodeGetHugepageInfo(uint16_t node_index, NodeInf { uint16_t hp_sizes_cnt = SystemHugepageSizesCntPerNodeGet(node_index); if (hp_sizes_cnt == 0) { - SCLogError("hugepages not found for node %d", node_index); - return -SC_EEXIST; + SCLogInfo("hugepages not found for node %d", node_index); + return -SC_ENOENT; } uint32_t *hp_sizes = SCCalloc(hp_sizes_cnt, sizeof(*hp_sizes)); if (hp_sizes == NULL) { @@ -202,10 +208,9 @@ static int16_t SystemHugepagePerNodeGetHugepageInfo(uint16_t node_index, NodeInf node->num_hugepage_sizes = hp_sizes_cnt; int16_t ret = 0; -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - ret = SystemHugepagePerNodeGetHugepageInfoLinux( - node->hugepages, hp_sizes, node->num_hugepage_sizes, node_index); -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ + if (SystemHugepageDetermineOS() == OS_LINUX_SYS_DEVICES) + ret = SystemHugepagePerNodeGetHugepageInfoLinux( + node->hugepages, hp_sizes, node->num_hugepage_sizes, node_index); SCFree(hp_sizes); return ret; @@ -217,9 +222,8 @@ static int16_t SystemHugepagePerNodeGetHugepageInfo(uint16_t node_index, NodeInf */ static uint16_t SystemNodeCountGet(void) { -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - return SystemNodeCountGetLinux(); -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ + if (SystemHugepageDetermineOS() == OS_LINUX_SYS_DEVICES) + return SystemNodeCountGetLinux(); return 0; } @@ -229,9 +233,8 @@ static uint16_t SystemNodeCountGet(void) */ static uint16_t SystemHugepageSizesCntPerNodeGet(uint16_t node_index) { -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - return SystemHugepageSizesCntPerNodeGetLinux(node_index); -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ + if (SystemHugepageDetermineOS() == OS_LINUX_SYS_DEVICES) + return SystemHugepageSizesCntPerNodeGetLinux(node_index); return 0; } @@ -245,9 +248,8 @@ static uint16_t SystemHugepageSizesCntPerNodeGet(uint16_t node_index) static void SystemHugepagePerNodeGetHugepageSizes( uint16_t node_index, uint16_t hp_sizes_cnt, uint32_t *hp_sizes) { -#if !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun - return SystemHugepagePerNodeGetHugepageSizesLinux(node_index, hp_sizes_cnt, hp_sizes); -#endif /* !defined __CYGWIN__ && !defined OS_WIN32 && !defined __OpenBSD__ && !defined sun */ + if (SystemHugepageDetermineOS() == OS_LINUX_SYS_DEVICES) + SystemHugepagePerNodeGetHugepageSizesLinux(node_index, hp_sizes_cnt, hp_sizes); } static HugepageInfo *SystemHugepageHugepageInfoCreate(uint16_t hp_size_cnt) @@ -325,7 +327,7 @@ SystemHugepageSnapshot *SystemHugepageSnapshotCreate(void) uint16_t node_cnt = SystemNodeCountGet(); if (node_cnt == 0) { - SCLogError("failed to obtain number of NUMA nodes in the system"); + SCLogInfo("hugepage snapshot failed - cannot obtain number of NUMA nodes in the system"); return NULL; } NodeInfo *nodes = SCCalloc(node_cnt, sizeof(*nodes)); @@ -386,7 +388,8 @@ void SystemHugepageEvaluateHugepages(SystemHugepageSnapshot *pre_s, SystemHugepa SCLogWarning( "Hugepage usage decreased while it should only increase/stay the same"); } else if (prerun_hp->free > 0 && prerun_hp->free == postrun_hp->free) { - SCLogPerf("Hugepages on NUMA node %u are unused and can be deallocated", i); + SCLogPerf("%ukB hugepages on NUMA node %u are unused and can be deallocated", + postrun_hp->size_kb, i); } else { // assumes this is an active NUMA node because at least some hugepages were // used // speculative hint only for 2048kB pages as e.g. 1 GB pages can leave a lot of room diff --git a/src/util-streaming-buffer.c b/src/util-streaming-buffer.c index 7608b50..6ff4f43 100644 --- a/src/util-streaming-buffer.c +++ b/src/util-streaming-buffer.c @@ -842,16 +842,11 @@ static inline void StreamingBufferSlideToOffsetWithRegions( r = next; } SCLogDebug("to_shift %p", to_shift); - } else { - to_shift = &sb->region; - SCLogDebug("shift start region %p", to_shift); - } - // this region is main, or will xfer its buffer to main - if (to_shift) { - SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset, - to_shift->buf, to_shift->buf_size, to_shift->buf_offset); - if (to_shift != &sb->region) { + // this region is main, or will xfer its buffer to main + if (to_shift && to_shift != &sb->region) { + SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset, + to_shift->buf, to_shift->buf_size, to_shift->buf_offset); DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL); sb->region.buf = to_shift->buf; @@ -860,12 +855,20 @@ static inline void StreamingBufferSlideToOffsetWithRegions( sb->region.buf_size = to_shift->buf_size; sb->region.next = to_shift->next; + BUG_ON(to_shift == &sb->region); FREE(cfg, to_shift, sizeof(*to_shift)); to_shift = &sb->region; sb->regions--; DEBUG_VALIDATE_BUG_ON(sb->regions == 0); } + } else { + to_shift = &sb->region; + SCLogDebug("shift start region %p", to_shift); + } + + // this region is main, or will xfer its buffer to main + if (to_shift) { // Do the shift. If new region is exactly at the slide offset we can skip this. DEBUG_VALIDATE_BUG_ON(to_shift->stream_offset > slide_offset); const uint32_t s = slide_offset - to_shift->stream_offset; |