/* Copyright (C) 2007-2022 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free * Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * version 2 along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA * 02110-1301, USA. */ /** * \file * * \author Gurvinder Singh * \author Victor Julien * * Reference: * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007 * */ #include "suricata-common.h" #include "suricata.h" #include "detect.h" #include "flow.h" #include "threads.h" #include "conf.h" #include "flow-util.h" #include "threadvars.h" #include "tm-threads.h" #include "util-pool.h" #include "util-unittest.h" #include "util-print.h" #include "util-host-os-info.h" #include "util-unittest-helper.h" #include "util-byte.h" #include "util-device.h" #include "stream-tcp.h" #include "stream-tcp-private.h" #include "stream-tcp-cache.h" #include "stream-tcp-reassemble.h" #include "stream-tcp-inline.h" #include "stream-tcp-list.h" #include "stream-tcp-util.h" #include "stream.h" #include "util-debug.h" #include "app-layer-protos.h" #include "app-layer.h" #include "app-layer-events.h" #include "app-layer-parser.h" #include "app-layer-frames.h" #include "detect-engine-state.h" #include "util-profiling.h" #include "util-validate.h" #include "util-exception-policy.h" #ifdef DEBUG static SCMutex segment_pool_memuse_mutex; static uint64_t segment_pool_memuse = 0; static uint64_t segment_pool_memcnt = 0; #endif thread_local uint64_t t_pcapcnt = UINT64_MAX; PoolThread *segment_thread_pool = NULL; /* init only, protect initializing and growing pool */ static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER; /* Memory use counter */ SC_ATOMIC_DECLARE(uint64_t, ra_memuse); static int g_tcp_session_dump_enabled = 0; inline bool IsTcpSessionDumpingEnabled(void) { return g_tcp_session_dump_enabled == 1; } void EnableTcpSessionDumping(void) { g_tcp_session_dump_enabled = 1; } /* prototypes */ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *); void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t); void StreamTcpReassembleInitMemuse(void) { SC_ATOMIC_INIT(ra_memuse); } /** * \brief Function to Increment the memory usage counter for the TCP reassembly * segments * * \param size Size of the TCP segment and its payload length memory allocated */ void StreamTcpReassembleIncrMemuse(uint64_t size) { (void) SC_ATOMIC_ADD(ra_memuse, size); SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size); return; } /** * \brief Function to Decrease the memory usage counter for the TCP reassembly * segments * * \param size Size of the TCP segment and its payload length memory allocated */ void StreamTcpReassembleDecrMemuse(uint64_t size) { #ifdef UNITTESTS uint64_t presize = SC_ATOMIC_GET(ra_memuse); if (RunmodeIsUnittests()) { BUG_ON(presize > UINT_MAX); } #endif (void) SC_ATOMIC_SUB(ra_memuse, size); #ifdef UNITTESTS if (RunmodeIsUnittests()) { uint64_t postsize = SC_ATOMIC_GET(ra_memuse); BUG_ON(postsize > presize); } #endif SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size); return; } uint64_t StreamTcpReassembleMemuseGlobalCounter(void) { uint64_t smemuse = SC_ATOMIC_GET(ra_memuse); return smemuse; } /** * \brief Function to Check the reassembly memory usage counter against the * allowed max memory usage for TCP segments. * * \param size Size of the TCP segment and its payload length memory allocated * \retval 1 if in bounds * \retval 0 if not in bounds */ int StreamTcpReassembleCheckMemcap(uint64_t size) { #ifdef DEBUG if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX && g_eps_stream_reassembly_memcap == t_pcapcnt))) { SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt); return 0; } #endif uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap); if (memcapcopy == 0 || (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy) return 1; return 0; } /** * \brief Update memcap value * * \param size new memcap value */ int StreamTcpReassembleSetMemcap(uint64_t size) { if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) { SC_ATOMIC_SET(stream_config.reassembly_memcap, size); return 1; } return 0; } /** * \brief Return memcap value * * \return memcap memcap value */ uint64_t StreamTcpReassembleGetMemcap(void) { uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap); return memcapcopy; } /* memory functions for the streaming buffer API */ /* void *(*Calloc)(size_t n, size_t size); */ static void *ReassembleCalloc(size_t n, size_t size) { if (StreamTcpReassembleCheckMemcap(n * size) == 0) { sc_errno = SC_ELIMIT; return NULL; } void *ptr = SCCalloc(n, size); if (ptr == NULL) { sc_errno = SC_ENOMEM; return NULL; } StreamTcpReassembleIncrMemuse(n * size); return ptr; } /* void *(*Realloc)(void *ptr, size_t orig_size, size_t size); */ void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size) { if (size > orig_size) { if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) { SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap)); sc_errno = SC_ELIMIT; return NULL; } } void *nptr = SCRealloc(optr, size); if (nptr == NULL) { SCLogDebug("realloc fail"); sc_errno = SC_ENOMEM; return NULL; } if (size > orig_size) { StreamTcpReassembleIncrMemuse(size - orig_size); } else { StreamTcpReassembleDecrMemuse(orig_size - size); } return nptr; } /* void (*Free)(void *ptr, size_t size); */ static void ReassembleFree(void *ptr, size_t size) { SCFree(ptr); StreamTcpReassembleDecrMemuse(size); } /** \brief alloc a tcp segment pool entry */ static void *TcpSegmentPoolAlloc(void) { SCLogDebug("segment alloc"); if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) { return NULL; } TcpSegment *seg = NULL; seg = SCMalloc(sizeof (TcpSegment)); if (unlikely(seg == NULL)) return NULL; if (IsTcpSessionDumpingEnabled()) { uint32_t memuse = sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE; if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) { SCFree(seg); return NULL; } seg->pcap_hdr_storage = SCCalloc(1, sizeof(TcpSegmentPcapHdrStorage)); if (seg->pcap_hdr_storage == NULL) { SCLogError("Unable to allocate memory for " "TcpSegmentPcapHdrStorage"); SCFree(seg); return NULL; } else { seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE; seg->pcap_hdr_storage->pkt_hdr = SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE); if (seg->pcap_hdr_storage->pkt_hdr == NULL) { SCLogError("Unable to allocate memory for " "packet header data within " "TcpSegmentPcapHdrStorage"); SCFree(seg->pcap_hdr_storage); SCFree(seg); return NULL; } } StreamTcpReassembleIncrMemuse(memuse); } else { seg->pcap_hdr_storage = NULL; } return seg; } static int TcpSegmentPoolInit(void *data, void *initdata) { TcpSegment *seg = (TcpSegment *) data; TcpSegmentPcapHdrStorage *pcap_hdr; pcap_hdr = seg->pcap_hdr_storage; /* do this before the can bail, so TcpSegmentPoolCleanup * won't have uninitialized memory to consider. */ memset(seg, 0, sizeof (TcpSegment)); if (IsTcpSessionDumpingEnabled()) { uint32_t memuse = sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE; seg->pcap_hdr_storage = pcap_hdr; if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) { return 0; } StreamTcpReassembleIncrMemuse(memuse); } else { if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) { return 0; } } #ifdef DEBUG SCMutexLock(&segment_pool_memuse_mutex); segment_pool_memuse += sizeof(TcpSegment); segment_pool_memcnt++; SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt); SCMutexUnlock(&segment_pool_memuse_mutex); #endif StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment)); return 1; } /** \brief clean up a tcp segment pool entry */ static void TcpSegmentPoolCleanup(void *ptr) { if (ptr == NULL) return; TcpSegment *seg = (TcpSegment *)ptr; if (seg && seg->pcap_hdr_storage) { if (seg->pcap_hdr_storage->pkt_hdr) { SCFree(seg->pcap_hdr_storage->pkt_hdr); StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen); } SCFree(seg->pcap_hdr_storage); seg->pcap_hdr_storage = NULL; StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage)); } StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment)); #ifdef DEBUG SCMutexLock(&segment_pool_memuse_mutex); segment_pool_memuse -= sizeof(TcpSegment); segment_pool_memcnt--; SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt); SCMutexUnlock(&segment_pool_memuse_mutex); #endif } /** * \brief Function to return the segment back to the pool. * * \param seg Segment which will be returned back to the pool. */ void StreamTcpSegmentReturntoPool(TcpSegment *seg) { if (seg == NULL) return; if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) { seg->pcap_hdr_storage->pktlen = 0; } StreamTcpThreadCacheReturnSegment(seg); } /** * \brief return all segments in this stream into the pool(s) * * \param stream the stream to cleanup */ void StreamTcpReturnStreamSegments (TcpStream *stream) { TcpSegment *seg = NULL, *safe = NULL; RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe) { RB_REMOVE(TCPSEG, &stream->seg_tree, seg); StreamTcpSegmentReturntoPool(seg); } } static inline uint64_t GetAbsLastAck(const TcpStream *stream) { if (STREAM_LASTACK_GT_BASESEQ(stream)) { return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq); } else { return STREAM_BASE_OFFSET(stream); } } uint64_t StreamTcpGetAcked(const TcpStream *stream) { return GetAbsLastAck(stream); } // may contain gaps uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof) { uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq; if (!eof && StreamTcpInlineMode() == FALSE) { right_edge = MIN(GetAbsLastAck(stream), right_edge); } return right_edge; } uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof) { uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb); if (!eof && StreamTcpInlineMode() == FALSE) { right_edge = MIN(GetAbsLastAck(stream), right_edge); } return right_edge; } #ifdef UNITTESTS /** \internal * \brief check if segments falls before stream 'offset' */ static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset) { if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset) return 1; return 0; } #endif /** \param f locked flow */ void StreamTcpDisableAppLayer(Flow *f) { if (f->protoctx == NULL) return; TcpSession *ssn = (TcpSession *)f->protoctx; StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server); StreamTcpDisableAppLayerReassembly(ssn); if (f->alparser) { AppLayerParserStateSetFlag(f->alparser, (APP_LAYER_PARSER_EOF_TS|APP_LAYER_PARSER_EOF_TC)); } } /** \param f locked flow */ int StreamTcpAppLayerIsDisabled(Flow *f) { if (f->protoctx == NULL || f->proto != IPPROTO_TCP) return 0; TcpSession *ssn = (TcpSession *)f->protoctx; return (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); } static int StreamTcpReassemblyConfig(bool quiet) { uint32_t segment_prealloc = 2048; ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc"); if (seg) { uint32_t prealloc = 0; if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) { SCLogError("segment-prealloc of " "%s is invalid", seg->val); return -1; } segment_prealloc = prealloc; } if (!quiet) SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc); stream_config.prealloc_segments = segment_prealloc; int overlap_diff_data = 0; (void)ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data); if (overlap_diff_data) { StreamTcpReassembleConfigEnableOverlapCheck(); } if (StreamTcpInlineMode() == TRUE) { StreamTcpReassembleConfigEnableOverlapCheck(); } uint16_t max_regions = 8; ConfNode *mr = ConfGetNode("stream.reassembly.max-regions"); if (mr) { uint16_t max_r = 0; if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) { SCLogError("max-regions %s is invalid", mr->val); return -1; } max_regions = max_r; } if (!quiet) SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions); stream_config.prealloc_segments = segment_prealloc; stream_config.sbcnf.buf_size = 2048; stream_config.sbcnf.max_regions = max_regions; stream_config.sbcnf.region_gap = STREAMING_BUFFER_REGION_GAP_DEFAULT; stream_config.sbcnf.Calloc = ReassembleCalloc; stream_config.sbcnf.Realloc = StreamTcpReassembleRealloc; stream_config.sbcnf.Free = ReassembleFree; return 0; } int StreamTcpReassembleInit(bool quiet) { /* init the memcap/use tracker */ StreamTcpReassembleInitMemuse(); if (StreamTcpReassemblyConfig(quiet) < 0) return -1; #ifdef DEBUG SCMutexInit(&segment_pool_memuse_mutex, NULL); #endif StatsRegisterGlobalCounter("tcp.reassembly_memuse", StreamTcpReassembleMemuseGlobalCounter); return 0; } void StreamTcpReassembleFree(bool quiet) { SCMutexLock(&segment_thread_pool_mutex); if (segment_thread_pool != NULL) { PoolThreadFree(segment_thread_pool); segment_thread_pool = NULL; } SCMutexUnlock(&segment_thread_pool_mutex); SCMutexDestroy(&segment_thread_pool_mutex); #ifdef DEBUG if (segment_pool_memuse > 0) SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "", segment_pool_memuse, segment_pool_memcnt); SCMutexDestroy(&segment_pool_memuse_mutex); #endif } TcpReassemblyThreadCtx *StreamTcpReassembleInitThreadCtx(ThreadVars *tv) { SCEnter(); TcpReassemblyThreadCtx *ra_ctx = SCMalloc(sizeof(TcpReassemblyThreadCtx)); if (unlikely(ra_ctx == NULL)) return NULL; memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx)); ra_ctx->app_tctx = AppLayerGetCtxThread(tv); SCMutexLock(&segment_thread_pool_mutex); if (segment_thread_pool == NULL) { segment_thread_pool = PoolThreadInit(1, /* thread */ 0, /* unlimited */ stream_config.prealloc_segments, sizeof(TcpSegment), TcpSegmentPoolAlloc, TcpSegmentPoolInit, NULL, TcpSegmentPoolCleanup, NULL); ra_ctx->segment_thread_pool_id = 0; SCLogDebug("pool size %d, thread segment_thread_pool_id %d", PoolThreadSize(segment_thread_pool), ra_ctx->segment_thread_pool_id); } else { /* grow segment_thread_pool until we have an element for our thread id */ ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool); SCLogDebug("pool size %d, thread segment_thread_pool_id %d", PoolThreadSize(segment_thread_pool), ra_ctx->segment_thread_pool_id); } SCMutexUnlock(&segment_thread_pool_mutex); if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) { SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?"); StreamTcpReassembleFreeThreadCtx(ra_ctx); SCReturnPtr(NULL, "TcpReassemblyThreadCtx"); } SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx"); } void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx) { SCEnter(); StreamTcpThreadCacheCleanup(); if (ra_ctx) { AppLayerDestroyCtxThread(ra_ctx->app_tctx); SCFree(ra_ctx); } SCReturn; } /** * \brief check if stream in pkt direction has depth reached * * \param p packet with *LOCKED* flow * * \retval 1 stream has depth reached * \retval 0 stream does not have depth reached */ int StreamTcpReassembleDepthReached(Packet *p) { if (p->flow != NULL && p->flow->protoctx != NULL) { TcpSession *ssn = p->flow->protoctx; TcpStream *stream; if (p->flowflags & FLOW_PKT_TOSERVER) { stream = &ssn->client; } else { stream = &ssn->server; } return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0; } return 0; } /** * \internal * \brief Function to Check the reassembly depth valuer against the * allowed max depth of the stream reassembly for TCP streams. * * \param stream stream direction * \param seq sequence number where "size" starts * \param size size of the segment that is added * * \retval size Part of the size that fits in the depth, 0 if none */ static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream, uint32_t seq, uint32_t size) { SCEnter(); /* if the configured depth value is 0, it means there is no limit on reassembly depth. Otherwise carry on my boy ;) */ if (ssn->reassembly_depth == 0) { SCReturnUInt(size); } /* if the final flag is set, we're not accepting anymore */ if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { SCReturnUInt(0); } uint64_t seg_depth; if (SEQ_GT(stream->base_seq, seq)) { if (SEQ_LEQ(seq+size, stream->base_seq)) { SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u", stream->base_seq, seq, seq+size); SCReturnUInt(0); } seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq); } else { seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq); } /* if the base_seq has moved passed the depth window we stop * checking and just reject the rest of the packets including * retransmissions. Saves us the hassle of dealing with sequence * wraps as well */ SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size), stream->base_seq, seg_depth, ssn->reassembly_depth); if (seg_depth > (uint64_t)ssn->reassembly_depth) { SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED"); stream->flags |= STREAMTCP_STREAM_FLAG_DEPTH_REACHED; SCReturnUInt(0); } SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED"); SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth); #if 0 SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32, (stream->base_seq_offset + stream->base_seq + size), (stream->isn + ssn->reassembly_depth)); #endif if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) { /* packet (partly?) fits the depth window */ if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) { /* complete fit */ SCReturnUInt(size); } else { stream->flags |= STREAMTCP_STREAM_FLAG_DEPTH_REACHED; /* partial fit, return only what fits */ uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq; DEBUG_VALIDATE_BUG_ON(part > size); if (part > size) part = size; SCReturnUInt(part); } } SCReturnUInt(0); } uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream) { if (RB_EMPTY(&stream->sb.sbb_tree)) { if (stream->sb.region.stream_offset != 0) return 0; return stream->sb.region.buf_offset; } else { DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL); DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0); return stream->sb.sbb_size; } } /** * \brief Insert a packets TCP data into the stream reassembly engine. * * \retval 0 good segment, as far as we checked. * \retval -1 insert failure due to memcap * * If the retval is 0 the segment is inserted correctly, or overlap is handled, * or it wasn't added because of reassembly depth. * */ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); if (ssn->data_first_seen_dir == 0) { if (PKT_IS_TOSERVER(p)) { ssn->data_first_seen_dir = STREAM_TOSERVER; } else { ssn->data_first_seen_dir = STREAM_TOCLIENT; } } /* If the OS policy is not set then set the OS policy for this stream */ if (stream->os_policy == 0) { StreamTcpSetOSPolicy(stream, p); } if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) && (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)) { SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn); SCReturnInt(0); } /* If we have reached the defined depth for either of the stream, then stop reassembling the TCP session */ uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len); SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size); if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { StreamTcpSetEvent(p, STREAM_REASSEMBLY_DEPTH_REACHED); /* increment stream depth counter */ StatsIncr(tv, ra_ctx->counter_tcp_stream_depth); p->app_update_direction = UPDATE_DIR_PACKET; } if (size == 0) { SCLogDebug("ssn %p: depth reached, not reassembling", ssn); SCReturnInt(0); } DEBUG_VALIDATE_BUG_ON(size > p->payload_len); if (size > p->payload_len) size = p->payload_len; TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx); if (seg == NULL) { SCLogDebug("segment_pool is empty"); StreamTcpSetEvent(p, STREAM_REASSEMBLY_NO_SEGMENT); ssn->flags |= STREAMTCP_FLAG_LOSSY_BE_LIBERAL; SCReturnInt(-1); } DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX); TCP_SEG_LEN(seg) = (uint16_t)size; seg->seq = TCP_GET_SEQ(p); /* HACK: for TFO SYN packets the seq for data starts at + 1 */ if (TCP_HAS_TFO(p) && p->payload_len && (p->tcph->th_flags & TH_SYN)) seg->seq += 1; /* proto detection skipped, but now we do get data. Set event. */ if (RB_EMPTY(&stream->seg_tree) && stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED) { AppLayerDecoderEventsSetEventRaw(&p->app_layer_events, APPLAYER_PROTO_DETECTION_SKIPPED); } int r = StreamTcpReassembleInsertSegment( tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len); if (r < 0) { if (r == -SC_ENOMEM) { ssn->flags |= STREAMTCP_FLAG_LOSSY_BE_LIBERAL; } SCLogDebug("StreamTcpReassembleInsertSegment failed"); SCReturnInt(-1); } SCReturnInt(0); } static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream, Packet *p) { uint8_t flag = 0; if (!(stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED)) { flag |= STREAM_START; } if (ssn->state == TCP_CLOSED) { flag |= STREAM_EOF; } if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) { flag |= STREAM_MIDSTREAM; } if (p->flags & PKT_PSEUDO_STREAM_END) { flag |= STREAM_EOF; } if (&ssn->client == stream) { flag |= STREAM_TOSERVER; } else { flag |= STREAM_TOCLIENT; } if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { flag |= STREAM_DEPTH; } return flag; } /** * \brief Check the minimum size limits for reassembly. * * \retval 0 don't reassemble yet * \retval 1 do reassemble */ static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn, const TcpStream *stream, const Packet *p) { SCEnter(); /* if any of these flags is set we always inspect immediately */ #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \ ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \ | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \ | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) if (stream->flags & STREAMTCP_STREAM_FLAG_FLUSH_FLAGS) { if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED " "is set, so not expecting any new data segments"); } if (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) { SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set"); } if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) { SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, " "so no new segments will be considered"); } SCReturnInt(1); } #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS /* some states mean we reassemble no matter how much data we have */ if (ssn->state > TCP_TIME_WAIT) SCReturnInt(1); if (p->flags & PKT_PSEUDO_STREAM_END) SCReturnInt(1); const uint64_t last_ack_abs = GetAbsLastAck(stream); int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream); int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size : (int64_t)stream_config.reassembly_toclient_chunk_size; /* check if we have enough data to do raw reassembly */ if (chunk_size <= diff) { SCReturnInt(1); } else { SCLogDebug("%s min chunk len not yet reached: " "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < " "%" PRIi64, PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq, (stream->last_ack - stream->base_seq), chunk_size); SCReturnInt(0); } SCReturnInt(0); } /** * \brief see what if any work the TCP session still needs */ uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction) { const TcpStream *stream = NULL; #ifdef DEBUG const char *dirstr = NULL; #endif if (direction == STREAM_TOSERVER) { stream = &ssn->client; #ifdef DEBUG dirstr = "client"; #endif } else { stream = &ssn->server; #ifdef DEBUG dirstr = "server"; #endif } int use_app = 1; int use_raw = 1; if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) { // app is dead use_app = 0; } if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) { // raw is dead use_raw = 0; } if (use_raw) { const uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq; SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64 " (use: %s). Stream right edge: %" PRIu64, dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no", STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge); if (right_edge > STREAM_RAW_PROGRESS(stream)) { SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr); return STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION; } } if (use_app) { const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb); SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64 " (use: %s). Stream right edge: %" PRIu64, dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no", STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge); if (right_edge > STREAM_APP_PROGRESS(stream)) { SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr); return STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION; } } SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr); return STREAM_HAS_UNPROCESSED_SEGMENTS_NONE; } #ifdef DEBUG static uint64_t GetStreamSize(TcpStream *stream) { if (stream) { uint64_t size = 0; uint32_t cnt = 0; uint64_t last_ack_abs = GetAbsLastAck(stream); uint64_t last_re = 0; SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset); TcpSegment *seg; RB_FOREACH(seg, TCPSEG, &stream->seg_tree) { const uint64_t seg_abs = STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq); if (last_re != 0 && last_re < seg_abs) { const char *gacked = NULL; if (last_ack_abs >= seg_abs) { gacked = "fully ack'd"; } else if (last_ack_abs > last_re) { gacked = "partly ack'd"; } else { gacked = "not yet ack'd"; } SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked); } const char *acked = NULL; if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) { acked = "fully ack'd"; } else if (last_ack_abs > seg_abs) { acked = "partly ack'd"; } else { acked = "not yet ack'd"; } SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64 ") ack:%s", cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg), seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked); last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg); cnt++; size += (uint64_t)TCP_SEG_LEN(seg); } SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt); return size; } return (uint64_t)0; } static void GetSessionSize(TcpSession *ssn, Packet *p) { uint64_t size = 0; if (ssn) { size = GetStreamSize(&ssn->client); size += GetStreamSize(&ssn->server); //if (size > 900000) // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt); SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt); } } #endif static StreamingBufferBlock *GetBlock(const StreamingBuffer *sb, const uint64_t offset) { StreamingBufferBlock *blk = sb->head; if (blk == NULL) return NULL; for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) { if (blk->offset >= offset) return blk; else if ((blk->offset + blk->len) > offset) { return blk; } } return NULL; } static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk) { StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk); if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) && GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) { return true; } return false; } /** \internal * * Get buffer, or first part of the buffer if data gaps exist. * * \brief get stream data from offset * \param offset stream offset * \param check_for_gap check if there is a gap ahead. Optional as it is only * needed for app-layer incomplete support. * \retval bool pkt loss ahead */ static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset, const bool check_for_gap) { const uint8_t *mydata; uint32_t mydata_len; bool gap_ahead = false; if (RB_EMPTY(&stream->sb.sbb_tree)) { SCLogDebug("getting one blob"); StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset); *data = mydata; *data_len = mydata_len; } else { SCLogDebug("block mode"); StreamingBufferBlock *blk = GetBlock(&stream->sb, offset); if (blk == NULL) { *data = NULL; *data_len = 0; return false; } SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len); /* block at expected offset */ if (blk->offset == offset) { SCLogDebug("blk at offset"); StreamingBufferSBBGetData(&stream->sb, blk, data, data_len); BUG_ON(blk->len != *data_len); gap_ahead = check_for_gap && GapAhead(stream, blk); /* block past out offset */ } else if (blk->offset > offset) { SCLogDebug("gap, want data at offset %"PRIu64", " "got data at %"PRIu64". GAP of size %"PRIu64, offset, blk->offset, blk->offset - offset); *data = NULL; *data_len = blk->offset - offset; /* block starts before offset, but ends after */ } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) { SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u", offset, blk->offset, blk->len); StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset); SCLogDebug("data %p, data_len %u", *data, *data_len); gap_ahead = check_for_gap && GapAhead(stream, blk); } else { *data = NULL; *data_len = 0; } } return gap_ahead; } /** \internal * \brief check to see if we should declare a GAP * Call this when the app layer didn't get data at the requested * offset. */ static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p) { const uint64_t app_progress = STREAM_APP_PROGRESS(stream); const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0; const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded; SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs); SCLogDebug("next_seq %u", stream->next_seq); /* if last_ack_abs is beyond the app_progress data that we haven't seen * has been ack'd. This looks like a GAP. */ if (last_ack_abs > app_progress) { /* however, we can accept ACKs a bit too liberally. If last_ack * is beyond next_seq, we only consider it a gap now if we do * already have data beyond the gap. */ if (SEQ_GT(stream->last_ack, stream->next_seq)) { if (RB_EMPTY(&stream->sb.sbb_tree)) { SCLogDebug("packet %" PRIu64 ": no GAP. " "next_seq %u < last_ack %u, but no data in list", p->pcap_cnt, stream->next_seq, stream->last_ack); return false; } else { const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq); const StreamingBufferBlock *blk = stream->sb.head; if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) { /* ack'd data after the gap */ SCLogDebug("packet %" PRIu64 ": GAP. " "next_seq %u < last_ack %u, but ACK'd data beyond gap.", p->pcap_cnt, stream->next_seq, stream->last_ack); return true; } } } SCLogDebug("packet %" PRIu64 ": GAP! " "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", " "but we have no data.", p->pcap_cnt, last_ack_abs, app_progress); return true; } SCLogDebug("packet %"PRIu64": no GAP. " "last_ack_abs %"PRIu64" <= app_progress %"PRIu64, p->pcap_cnt, last_ack_abs, app_progress); return false; } static inline uint32_t AdjustToAcked(const Packet *p, const TcpSession *ssn, const TcpStream *stream, const uint64_t app_progress, const uint32_t data_len) { uint32_t adjusted = data_len; /* get window of data that is acked */ if (StreamTcpInlineMode() == FALSE) { SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state)); if (data_len == 0 || ((ssn->state < TCP_CLOSED || (ssn->state == TCP_CLOSED && (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) && (p->flags & PKT_PSEUDO_STREAM_END))) { // fall through, we use all available data } else { const uint64_t last_ack_abs = GetAbsLastAck(stream); DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs); /* see if the buffer contains unack'd data as well */ if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) { uint32_t check = data_len; adjusted = last_ack_abs - app_progress; BUG_ON(adjusted > check); SCLogDebug("data len adjusted to %u to make sure only ACK'd " "data is considered", adjusted); } } } return adjusted; } /** \internal * \brief get stream buffer and update the app-layer * \param stream pointer to pointer as app-layer can switch flow dir * \retval 0 success */ static int ReassembleUpdateAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream **stream, Packet *p, enum StreamUpdateDir dir) { uint64_t app_progress = STREAM_APP_PROGRESS(*stream); SCLogDebug("app progress %"PRIu64, app_progress); #ifdef DEBUG uint64_t last_ack_abs = GetAbsLastAck(*stream); SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs, (*stream)->base_seq); #endif const uint8_t *mydata; uint32_t mydata_len; bool last_was_gap = false; while (1) { const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p); bool check_for_gap_ahead = ((*stream)->data_required > 0); bool gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead); SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len); if (last_was_gap && mydata_len == 0) { break; } last_was_gap = false; /* make sure to only deal with ACK'd data */ mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len); DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX); if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) { SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len); int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len, StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP; StatsIncr(tv, ra_ctx->counter_tcp_reass_gap); ssn->flags |= STREAMTCP_FLAG_LOSSY_BE_LIBERAL; /* AppLayerHandleTCPData has likely updated progress. */ const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream)); app_progress = STREAM_APP_PROGRESS(*stream); /* a GAP also consumes 'data required'. TODO perhaps we can use * this to skip post GAP data until the start of a next record. */ if ((*stream)->data_required > 0) { if ((*stream)->data_required > mydata_len) { (*stream)->data_required -= mydata_len; } else { (*stream)->data_required = 0; } } if (r < 0) return 0; if (no_progress_update) break; last_was_gap = true; continue; } else if (flags & STREAM_DEPTH) { SCLogDebug("DEPTH"); // we're just called once with this flag, so make sure we pass it on if (mydata == NULL && mydata_len > 0) { mydata_len = 0; } } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) { SCLogDebug("GAP?1"); /* Possibly a gap, but no new data. */ if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) SCReturnInt(0); mydata = NULL; mydata_len = 0; SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len); break; } DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0); SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64, *stream, &(*stream)->sb, mydata_len, app_progress); if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) { SCLogDebug("GAP?2"); if (mydata_len < (*stream)->data_required) { SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead)); if (gap_ahead) { SCLogDebug("GAP while expecting more data (expect %u, gap size %u)", (*stream)->data_required, mydata_len); (*stream)->app_progress_rel += mydata_len; (*stream)->data_required -= mydata_len; // TODO send incomplete data to app-layer with special flag // indicating its all there is for this rec? } else { SCReturnInt(0); } app_progress = STREAM_APP_PROGRESS(*stream); continue; } } (*stream)->data_required = 0; SCLogDebug("parser"); /* update the app-layer */ (void)AppLayerHandleTCPData( tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, mydata_len, flags, dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); AppLayerFrameDump(p->flow); uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream); if (new_app_progress == app_progress || FlowChangeProto(p->flow)) break; app_progress = new_app_progress; if (flags & STREAM_DEPTH) break; } SCReturnInt(0); } /** * \brief Update the stream reassembly upon receiving a packet. * * For IDS mode, the stream is in the opposite direction of the packet, * as the ACK-packet is ACK'ing the stream. * * One of the utilities call by this function AppLayerHandleTCPData(), * has a feature where it will call this very same function for the * stream opposing the stream it is called with. This shouldn't cause * any issues, since processing of each stream is independent of the * other stream. */ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir) { SCEnter(); /* this function can be directly called by app layer protocol * detection. */ if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) || (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { SCLogDebug("stream no reassembly flag set or app-layer disabled."); SCReturnInt(0); } #ifdef DEBUG SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree)); GetSessionSize(ssn, p); #endif /* if no segments are in the list or all are already processed, * and state is beyond established, we send an empty msg */ if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream)) { /* send an empty EOF msg if we have no segments but TCP state * is beyond ESTABLISHED */ if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) { SCLogDebug("sending empty eof message"); /* send EOF to app layer */ AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, StreamGetAppLayerFlags(ssn, stream, p), dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); SCReturnInt(0); } } /* with all that out of the way, lets update the app-layer */ return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir); } /** \internal * \brief get stream data from offset * \param offset stream offset */ static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset) { const uint8_t *mydata; uint32_t mydata_len; if (RB_EMPTY(&stream->sb.sbb_tree)) { SCLogDebug("getting one blob for offset %"PRIu64, offset); uint64_t roffset = offset; if (offset) StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset); else { StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset); } *data = mydata; *data_len = mydata_len; *data_offset = roffset; } else { SCLogDebug("multiblob %s. Want offset %"PRIu64, *iter == NULL ? "starting" : "continuing", offset); if (*iter == NULL) { StreamingBufferBlock key = { .offset = offset, .len = 0 }; *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key); SCLogDebug("*iter %p", *iter); } if (*iter == NULL) { SCLogDebug("no data"); *data = NULL; *data_len = 0; *data_offset = 0; return 0; } SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len); StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len); SCLogDebug("mydata %p", mydata); if ((*iter)->offset < offset) { uint64_t delta = offset - (*iter)->offset; if (delta < mydata_len) { *data = mydata + delta; *data_len = mydata_len - delta; *data_offset = offset; } else { SCLogDebug("no data (yet)"); *data = NULL; *data_len = 0; *data_offset = 0; } } else { *data = mydata; *data_len = mydata_len; *data_offset = (*iter)->offset; } *iter = SBB_RB_NEXT(*iter); SCLogDebug("*iter %p", *iter); } return 0; } /** \brief does the stream engine have data to inspect? * * Returns true if there is data to inspect. In IDS case this is * about ACK'd data in the packet's direction. * * In the IPS case this is about the packet itself. */ bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p) { TcpStream *stream; if (PKT_IS_TOSERVER(p)) { stream = &ssn->client; } else { stream = &ssn->server; } if (RB_EMPTY(&stream->seg_tree)) { return false; } if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY| STREAMTCP_STREAM_FLAG_DISABLE_RAW)) return false; if (StreamTcpInlineMode() == FALSE) { const uint64_t segs_re_abs = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq; if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) { return false; } if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) { return true; } } else { if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) { return true; } } return false; } /** \brief update stream engine after detection * * Tasked with progressing the 'progress' for Raw reassembly. * 2 main scenario's: * 1. progress is != 0, so we use this * 2. progress is 0, meaning the detect engine didn't touch * raw at all. In this case we need to look into progressing * raw anyway. * * Additionally, this function is tasked with disabling raw * reassembly if the app-layer requested to disable it. */ void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress) { TcpStream *stream; if (PKT_IS_TOSERVER(p)) { stream = &ssn->client; } else { stream = &ssn->server; } if (progress > STREAM_RAW_PROGRESS(stream)) { uint32_t slide = progress - STREAM_RAW_PROGRESS(stream); stream->raw_progress_rel += slide; stream->flags &= ~STREAMTCP_STREAM_FLAG_TRIGGER_RAW; } else if (progress == 0) { uint64_t target; if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) { target = STREAM_APP_PROGRESS(stream); } else { target = GetAbsLastAck(stream); } if (target > STREAM_RAW_PROGRESS(stream)) { uint32_t slide = target - STREAM_RAW_PROGRESS(stream); stream->raw_progress_rel += slide; } stream->flags &= ~STREAMTCP_STREAM_FLAG_TRIGGER_RAW; } else { SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32, p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream), stream->window); } /* if we were told to accept no more raw data, we can mark raw as * disabled now. */ if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) { stream->flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW; SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, " "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn); } SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream)); } /** \internal * \brief get a buffer around the current packet and run the callback on it * * The inline/IPS scanning method takes the current payload and wraps it in * data from other segments. * * How much data is inspected is controlled by the available data, chunk_size * and the payload size of the packet. * * Large packets: if payload size is close to the chunk_size, where close is * defined as more than 67% of the chunk_size, a larger chunk_size will be * used: payload_len + 33% of the chunk_size. * If the payload size if equal to or bigger than the chunk_size, we use * payload len + 33% of the chunk size. */ static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out) { SCEnter(); int r = 0; TcpStream *stream; if (PKT_IS_TOSERVER(p)) { stream = &ssn->client; } else { stream = &ssn->server; } if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 || (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { *progress_out = STREAM_RAW_PROGRESS(stream); return 0; } uint32_t chunk_size = PKT_IS_TOSERVER(p) ? stream_config.reassembly_toserver_chunk_size : stream_config.reassembly_toclient_chunk_size; if (chunk_size <= p->payload_len) { chunk_size = p->payload_len + (chunk_size / 3); SCLogDebug("packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size); } else if (((chunk_size / 3 ) * 2) < p->payload_len) { chunk_size = p->payload_len + ((chunk_size / 3)); SCLogDebug("packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size); } uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq); uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len; SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64, packet_leftedge_abs, packet_rightedge_abs); const uint8_t *mydata = NULL; uint32_t mydata_len = 0; uint64_t mydata_offset = 0; /* simply return progress from the block we inspected. */ bool return_progress = false; if (RB_EMPTY(&stream->sb.sbb_tree)) { /* continues block */ StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset); return_progress = true; } else { SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs); /* find our block */ StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len }; StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key); if (sbb) { SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len); StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len); mydata_offset = sbb->offset; } } /* this can only happen if the segment insert of our current 'p' failed */ uint64_t mydata_rightedge_abs = mydata_offset + mydata_len; if ((mydata == NULL || mydata_len == 0) || /* no data */ (mydata_offset >= packet_rightedge_abs || /* data all to the right */ packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */ (packet_leftedge_abs < mydata_offset || /* data missing at the start */ packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */ { /* no data, or data is incomplete or wrong: use packet data */ mydata = p->payload; mydata_len = p->payload_len; mydata_offset = packet_leftedge_abs; //mydata_rightedge_abs = packet_rightedge_abs; } else { /* adjust buffer to match chunk_size */ SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len); if (mydata_len > chunk_size) { uint32_t excess = mydata_len - chunk_size; SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess); if (mydata_rightedge_abs == packet_rightedge_abs) { mydata += excess; mydata_len -= excess; mydata_offset += excess; SCLogDebug("cutting front of the buffer with %u", excess); } else if (mydata_offset == packet_leftedge_abs) { mydata_len -= excess; SCLogDebug("cutting tail of the buffer with %u", excess); } else { uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset); uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs); SCLogDebug("before %u after %u", before, after); if (after >= (chunk_size - p->payload_len) / 2) { // more trailing data than we need if (before >= (chunk_size - p->payload_len) / 2) { // also more heading data, divide evenly before = after = (chunk_size - p->payload_len) / 2; } else { // heading data is less than requested, give the // rest to the trailing data after = (chunk_size - p->payload_len) - before; } } else { // less trailing data than requested if (before >= (chunk_size - p->payload_len) / 2) { before = (chunk_size - p->payload_len) - after; } else { // both smaller than their requested size } } /* adjust the buffer */ uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before; uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after; DEBUG_VALIDATE_BUG_ON(skip > mydata_len); DEBUG_VALIDATE_BUG_ON(cut > mydata_len); DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len); mydata += skip; mydata_len -= (skip + cut); mydata_offset += skip; } } } /* run the callback */ r = Callback(cb_data, mydata, mydata_len, mydata_offset); BUG_ON(r < 0); if (return_progress) { *progress_out = (mydata_offset + mydata_len); } else { /* several blocks of data, so we need to be a bit more careful: * - if last_ack is beyond last progress, move progress forward to last_ack * - if our block matches or starts before last ack, return right edge of * our block. */ const uint64_t last_ack_abs = GetAbsLastAck(stream); SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs); if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) { if (mydata_offset > last_ack_abs) { /* gap between us and last ack, set progress to last ack */ *progress_out = last_ack_abs; } else { *progress_out = (mydata_offset + mydata_len); } } else { *progress_out = STREAM_RAW_PROGRESS(stream); } } return r; } /** \brief access 'raw' reassembly data. * * Access data as tracked by 'raw' tracker. Data is made available to * callback that is passed to this function. * * In the case of IDS the callback may be run multiple times if data * contains gaps. It will then be run for each block of data that is * continuous. * * The callback should give on of 2 return values: * - 0 ok * - 1 done * The value 1 will break the loop if there is a block list that is * inspected. * * This function will return the 'progress' value that has been * consumed until now. * * \param ssn tcp session * \param stream tcp stream * \param Callback the function pointer to the callback function * \param cb_data callback data * \param[in] progress_in progress to work from * \param[in] re right edge of data to consider * \param[out] progress_out absolute progress value of the data this * call handled. * \param eof we're wrapping up so inspect all data we have, incl unACKd * \param respect_inspect_depth use Stream::min_inspect_depth if set * * `respect_inspect_depth` is used to avoid useless inspection of too * much data. */ static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, const uint64_t re, uint64_t *progress_out, bool eof, bool respect_inspect_depth) { SCEnter(); int r = 0; StreamingBufferBlock *iter = NULL; uint64_t progress = progress_in; /* loop through available buffers. On no packet loss we'll have a single * iteration. On missing data we'll walk the blocks */ while (1) { const uint8_t *mydata; uint32_t mydata_len; uint64_t mydata_offset = 0; GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset); if (mydata_len == 0) { SCLogDebug("no data"); break; } //PrintRawDataFp(stdout, mydata, mydata_len); SCLogDebug("raw progress %"PRIu64, progress); SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64, stream, &stream->sb, mydata_len, progress); if (eof) { // inspect all remaining data, ack'd or not } else { if (re < progress) { SCLogDebug("nothing to do"); goto end; } SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress); SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len, re); /* see if the buffer contains unack'd data as well */ if (progress + mydata_len > re) { uint32_t check = mydata_len; mydata_len = re - progress; BUG_ON(check < mydata_len); SCLogDebug("data len adjusted to %u to make sure only ACK'd " "data is considered", mydata_len); } } if (mydata_len == 0) break; SCLogDebug("data %p len %u", mydata, mydata_len); /* we have data. */ r = Callback(cb_data, mydata, mydata_len, mydata_offset); BUG_ON(r < 0); if (mydata_offset == progress) { SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64, progress, mydata_len, progress_in + mydata_len); progress += mydata_len; SCLogDebug("raw progress now %"PRIu64, progress); /* data is beyond the progress we'd like, and before last ack. Gap. */ } else if (mydata_offset > progress && mydata_offset < re) { SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset); SCLogDebug("re %" PRIu64, re); progress = mydata_offset; SCLogDebug("raw progress now %"PRIu64, progress); } else { SCLogDebug("not increasing progress, data gap => mydata_offset " "%"PRIu64" != progress %"PRIu64, mydata_offset, progress); } if (iter == NULL || r == 1) break; } end: *progress_out = progress; return r; } int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof) { /* take app progress as the right edge of used data. */ const uint64_t app_progress = STREAM_APP_PROGRESS(stream); SCLogDebug("app_progress %" PRIu64, app_progress); uint64_t unused = 0; return StreamReassembleRawDo( ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false); } int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth) { /* handle inline separately as the logic is very different */ if (StreamTcpInlineMode() == TRUE) { return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out); } TcpStream *stream; if (PKT_IS_TOSERVER(p)) { stream = &ssn->client; } else { stream = &ssn->server; } if ((stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY|STREAMTCP_STREAM_FLAG_DISABLE_RAW)) || StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0) { *progress_out = STREAM_RAW_PROGRESS(stream); return 0; } uint64_t progress = STREAM_RAW_PROGRESS(stream); /* if the app layer triggered a flush, and we're supposed to * use a minimal inspect depth, we actually take the app progress * as that is the right edge of the data. Then we take the window * of 'min_inspect_depth' before that. */ SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s " "stream->min_inspect_depth %u", respect_inspect_depth ? "true" : "false", (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false", stream->min_inspect_depth); if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) && stream->min_inspect_depth) { progress = STREAM_APP_PROGRESS(stream); if (stream->min_inspect_depth >= progress) { progress = 0; } else { progress -= stream->min_inspect_depth; } SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream)); progress = MIN(progress, STREAM_RAW_PROGRESS(stream)); SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress " "%" PRIu64, progress); } SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW" : "(no trigger)"); /* absolute right edge of ack'd data */ const uint64_t last_ack_abs = GetAbsLastAck(stream); SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs); return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs, progress_out, (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth); } int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof) { if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) return 0; /* absolute right edge of ack'd data */ const uint64_t last_ack_abs = GetAbsLastAck(stream); SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs); return StreamReassembleRawDo( ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof, false); } /** \internal * \brief update app layer based on received ACK * * \retval r 0 on success, -1 on error */ static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0) SCReturnInt(-1); SCReturnInt(0); } int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); DEBUG_VALIDATE_BUG_ON(p->tcph == NULL); SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"", ssn, stream, p, p->payload_len); /* default IDS: update opposing side (triggered by ACK) */ enum StreamUpdateDir dir = UPDATE_DIR_OPPOSING; /* inline and stream end and flow timeout packets trigger same dir handling */ if (StreamTcpInlineMode()) { dir = UPDATE_DIR_PACKET; } else if (p->flags & PKT_PSEUDO_STREAM_END) { dir = UPDATE_DIR_PACKET; } else if (p->tcph->th_flags & TH_RST) { // accepted rst dir = UPDATE_DIR_PACKET; } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) { if (p->tcph->th_flags & TH_ACK) { dir = UPDATE_DIR_BOTH; } else { dir = UPDATE_DIR_PACKET; } } else if (ssn->state == TCP_CLOSED) { dir = UPDATE_DIR_BOTH; } /* handle ack received */ if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) { /* we need to update the opposing stream in * StreamTcpReassembleHandleSegmentUpdateACK */ TcpStream *opposing_stream = NULL; if (stream == &ssn->client) { opposing_stream = &ssn->server; } else { opposing_stream = &ssn->client; } const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0; if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) { SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error"); SCReturnInt(-1); } /* StreamTcpReassembleHandleSegmentUpdateACK * may swap content of ssn->server and ssn->client structures. * We have to continue with initial content of the stream in such case */ const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0; if (reversed_before_ack_handling != reversed_after_ack_handling) { SCLogDebug("TCP streams were swapped"); stream = opposing_stream; } } /* if this segment contains data, insert it */ if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) && (p->tcph->th_flags & TH_RST) == 0) { SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData"); if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) { SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error"); /* failure can only be because of memcap hit, so see if this should lead to a drop */ ExceptionPolicyApply( p, stream_config.reassembly_memcap_policy, PKT_DROP_REASON_STREAM_REASSEMBLY); SCReturnInt(-1); } SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt); p->flags |= PKT_STREAM_ADD; } else { SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:" " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s", ssn, stream, p->payload_len, (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false"); } /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag * was *just* set. In this case we trigger the AppLayer Truncate * logic, to inform the applayer no more data in this direction is * to be expected. */ if ((stream->flags & (STREAMTCP_STREAM_FLAG_DEPTH_REACHED|STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) == STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer"); if (dir != UPDATE_DIR_PACKET) { SCLogDebug("override: direction now UPDATE_DIR_PACKET so we " "can trigger Truncate"); dir = UPDATE_DIR_PACKET; } } /* in stream inline mode even if we have no data we call the reassembly * functions to handle EOF */ if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) { SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)", StreamTcpInlineMode()?"true":"false", (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false"); if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) { SCReturnInt(-1); } } SCReturnInt(0); } /** * \brief get a segment from the pool * * \retval seg Segment from the pool or NULL */ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx) { TcpSegment *seg = StreamTcpThreadCacheGetSegment(); if (seg) { StatsIncr(tv, ra_ctx->counter_tcp_segment_from_cache); memset(&seg->sbseg, 0, sizeof(seg->sbseg)); return seg; } seg = (TcpSegment *)PoolThreadGetById( segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id); SCLogDebug("seg we return is %p", seg); if (seg == NULL) { /* Increment the counter to show that we are not able to serve the segment request due to memcap limit */ StatsIncr(tv, ra_ctx->counter_tcp_segment_memcap); } else { memset(&seg->sbseg, 0, sizeof(seg->sbseg)); StatsIncr(tv, ra_ctx->counter_tcp_segment_from_pool); } return seg; } /** * \brief Trigger RAW stream reassembly * * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream * reassembly from the applayer, for example upon completion of a * HTTP request. * * It sets a flag in the stream so that the next Raw call will return * the data. * * \param ssn TcpSession */ void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction) { #ifdef DEBUG BUG_ON(ssn == NULL); #endif if (ssn != NULL) { if (direction == STREAM_TOSERVER) { ssn->client.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW; } else { ssn->server.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW; } SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn); } } void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth) { #ifdef DEBUG BUG_ON(ssn == NULL); #endif if (ssn != NULL) { if (direction == STREAM_TOSERVER) { ssn->client.min_inspect_depth = depth; SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth); } else { ssn->server.min_inspect_depth = depth; SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth); } } } #ifdef UNITTESTS /** unit tests and it's support functions below */ #define SET_ISN(stream, setseq) \ (stream)->isn = (setseq); \ (stream)->base_seq = (setseq) + 1 /** \brief The Function to create the packet with given payload, which is used * to test the reassembly of the engine. * * \param payload The variable used to store the payload contents of the * current packet. * \param value The value which current payload will have for this packet * \param payload_len The length of the filed payload for current packet. * \param len Length of the payload array */ void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value, uint8_t payload_len, uint8_t len) { uint8_t i; for (i = 0; i < payload_len; i++) payload[i] = value; for (; i < len; i++) payload = NULL; } /** \brief The Function Checks the reassembled stream contents against predefined * stream contents according to OS policy used. * * \param stream_policy Predefined value of stream for different OS policies * \param stream Reassembled stream returned from the reassembly functions */ int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream) { if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0) { //PrintRawDataFp(stdout, stream_policy, sp_size); return 0; } return 1; } static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len) { if (StreamingBufferCompareRawData(&stream->sb, data, data_len) == 0) { SCReturnInt(0); } SCLogInfo("OK"); PrintRawDataFp(stdout, data, data_len); return 1; } #define MISSED_START(isn) \ TcpReassemblyThreadCtx *ra_ctx = NULL; \ TcpSession ssn; \ ThreadVars tv; \ memset(&tv, 0, sizeof(tv)); \ \ StreamTcpUTInit(&ra_ctx); \ \ StreamTcpUTSetupSession(&ssn); \ StreamTcpUTSetupStream(&ssn.server, (isn)); \ StreamTcpUTSetupStream(&ssn.client, (isn)); \ \ TcpStream *stream = &ssn.client; #define MISSED_END \ StreamTcpUTClearSession(&ssn); \ StreamTcpUTDeinit(ra_ctx); \ PASS #define MISSED_STEP(seq, seg, seglen, buf, buflen) \ StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \ FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen)))); #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \ StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len); int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len) { int cnt = 0; uint64_t last_re = 0; StreamingBufferBlock *sbb = NULL; RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree) { if (sbb->offset != last_re) { // gap before us if (cnt == pos && last_re == offset && len == sbb->offset - last_re) { return 1; } cnt++; } last_re = sbb->offset + sbb->len; cnt++; } return 0; } int UTHCheckDataAtPosition( TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len); int UTHCheckDataAtPosition( TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len) { int cnt = 0; uint64_t last_re = 0; StreamingBufferBlock *sbb = NULL; RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree) { if (sbb->offset != last_re) { // gap before us cnt++; } if (cnt == pos && sbb->offset == offset) { const uint8_t *buf = NULL; uint32_t buf_len = 0; StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len); if (len == buf_len) { return (memcmp(data, buf, len) == 0); } } last_re = sbb->offset + sbb->len; cnt++; } return 0; } /** * \test Test the handling of packets missed by both IDS and the end host. * The packet is missed in the starting of the stream. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest25 (void) { MISSED_START(6); MISSED_ADD_PAYLOAD(10, "BB", 2); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1); MISSED_ADD_PAYLOAD(12, "CC", 2); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1); MISSED_STEP(7, "AAA", 3, "AAABBCC", 7); MISSED_END; PASS; } /** * \test Test the handling of packets missed by both IDS and the end host. * The packet is missed in the middle of the stream. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest26 (void) { MISSED_START(9); MISSED_STEP(10, "AAA", 3, "AAA", 3); MISSED_ADD_PAYLOAD(15, "CC", 2); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1); MISSED_STEP(13, "BB", 2, "AAABBCC", 7); MISSED_END; } /** * \test Test the handling of packets missed by both IDS and the end host. * The packet is missed in the end of the stream. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest27 (void) { MISSED_START(9); MISSED_STEP(10, "AAA", 3, "AAA", 3); MISSED_STEP(13, "BB", 2, "AAABB", 5); MISSED_STEP(15, "CC", 2, "AAABBCC", 7); MISSED_END; } /** * \test Test the handling of packets missed by IDS, but the end host has * received it and send the acknowledgment of it. The packet is missed * in the starting of the stream. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest28 (void) { MISSED_START(6); MISSED_ADD_PAYLOAD(10, "AAA", 3); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1); MISSED_ADD_PAYLOAD(13, "BB", 2); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1); ssn.state = TCP_TIME_WAIT; MISSED_ADD_PAYLOAD(15, "CC", 2); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1); MISSED_END; } /** * \test Test the handling of packets missed by IDS, but the end host has * received it and send the acknowledgment of it. The packet is missed * in the middle of the stream. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest29 (void) { MISSED_START(9); MISSED_STEP(10, "AAA", 3, "AAA", 3); ssn.state = TCP_TIME_WAIT; MISSED_ADD_PAYLOAD(15, "CC", 2); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1); FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1); FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1); MISSED_END; } static int StreamTcpReassembleTest33(void) { TcpSession ssn; Packet *p = PacketGetFromAlloc(); FAIL_IF(unlikely(p == NULL)); Flow f; TCPHdr tcph; TcpReassemblyThreadCtx *ra_ctx = NULL; ssn.client.os_policy = OS_POLICY_BSD; uint8_t packet[1460] = ""; StreamTcpUTInit(&ra_ctx); StreamTcpUTSetupSession(&ssn); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); FLOW_INITIALIZE(&f); f.protoctx = &ssn; f.proto = IPPROTO_TCP; p->src.family = AF_INET; p->dst.family = AF_INET; p->proto = IPPROTO_TCP; p->flow = &f; tcph.th_win = 5480; tcph.th_flags = TH_PUSH | TH_ACK; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; p->payload = packet; p->tcph->th_seq = htonl(10); p->tcph->th_ack = htonl(31); p->payload_len = 10; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(20); p->tcph->th_ack = htonl(31); p->payload_len = 10; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(40); p->tcph->th_ack = htonl(31); p->payload_len = 10; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(5); p->tcph->th_ack = htonl(31); p->payload_len = 30; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); SCFree(p); PASS; } static int StreamTcpReassembleTest34(void) { TcpSession ssn; Packet *p = PacketGetFromAlloc(); FAIL_IF(unlikely(p == NULL)); Flow f; TCPHdr tcph; TcpReassemblyThreadCtx *ra_ctx = NULL; ssn.client.os_policy = OS_POLICY_BSD; uint8_t packet[1460] = ""; StreamTcpUTInit(&ra_ctx); StreamTcpUTSetupSession(&ssn); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); FLOW_INITIALIZE(&f); f.protoctx = &ssn; f.proto = IPPROTO_TCP; p->src.family = AF_INET; p->dst.family = AF_INET; p->proto = IPPROTO_TCP; p->flow = &f; tcph.th_win = 5480; tcph.th_flags = TH_PUSH | TH_ACK; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; p->payload = packet; SET_ISN(&ssn.client, 857961230); p->tcph->th_seq = htonl(857961230); p->tcph->th_ack = htonl(31); p->payload_len = 304; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(857961534); p->tcph->th_ack = htonl(31); p->payload_len = 1460; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(857963582); p->tcph->th_ack = htonl(31); p->payload_len = 1460; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); p->tcph->th_seq = htonl(857960946); p->tcph->th_ack = htonl(31); p->payload_len = 1460; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); SCFree(p); PASS; } /** * \test Test to make sure that we don't return the segments until the app * layer proto has been detected and after that remove the processed * segments. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest39 (void) { Packet *p = PacketGetFromAlloc(); FAIL_IF(unlikely(p == NULL)); Flow f; ThreadVars tv; StreamTcpThread stt; TCPHdr tcph; PacketQueueNoLock pq; memset(&pq,0,sizeof(PacketQueueNoLock)); memset (&f, 0, sizeof(Flow)); memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (stt)); memset(&tcph, 0, sizeof (TCPHdr)); FLOW_INITIALIZE(&f); f.flags = FLOW_IPV4; f.proto = IPPROTO_TCP; p->flow = &f; p->tcph = &tcph; StreamTcpUTInit(&stt.ra_ctx); /* handshake */ tcph.th_win = htons(5480); tcph.th_flags = TH_SYN; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); TcpSession *ssn = (TcpSession *)f.protoctx; FAIL_IF_NULL(ssn); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != 0); /* handshake */ p->tcph->th_ack = htonl(1); p->tcph->th_flags = TH_SYN | TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != 0); /* handshake */ p->tcph->th_ack = htonl(1); p->tcph->th_seq = htonl(1); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != 0); /* partial request */ uint8_t request1[] = { 0x47, 0x45, }; p->tcph->th_ack = htonl(1); p->tcph->th_seq = htonl(1); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = sizeof(request1); p->payload = request1; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER); /* response ack against partial request */ p->tcph->th_ack = htonl(3); p->tcph->th_seq = htonl(1); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER); /* complete partial request */ uint8_t request2[] = { 0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30, 0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e, 0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20, 0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a }; p->tcph->th_ack = htonl(1); p->tcph->th_seq = htonl(3); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = sizeof(request2); p->payload = request2; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER); /* response - request ack */ uint8_t response[] = { 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31, 0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d, 0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46, 0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53, 0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20, 0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32, 0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69, 0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32, 0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a, 0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34, 0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31, 0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a, 0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61, 0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63, 0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61, 0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d, 0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20, 0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34, 0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58, 0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d, 0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c, 0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c, 0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31, 0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e }; p->tcph->th_ack = htonl(88); p->tcph->th_seq = htonl(1); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = sizeof(response); p->payload = response; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); /* response ack from request */ p->tcph->th_ack = htonl(328); p->tcph->th_seq = htonl(88); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response - acking */ p->tcph->th_ack = htonl(88); p->tcph->th_seq = htonl(328); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response ack from request */ p->tcph->th_ack = htonl(328); p->tcph->th_seq = htonl(88); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response - acking the request again*/ p->tcph->th_ack = htonl(88); p->tcph->th_seq = htonl(328); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /*** New Request ***/ /* partial request */ p->tcph->th_ack = htonl(328); p->tcph->th_seq = htonl(88); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = sizeof(request1); p->payload = request1; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response ack against partial request */ p->tcph->th_ack = htonl(90); p->tcph->th_seq = htonl(328); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* complete request */ p->tcph->th_ack = htonl(328); p->tcph->th_seq = htonl(90); p->tcph->th_flags = TH_PUSH | TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = sizeof(request2); p->payload = request2; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response ack against second partial request */ p->tcph->th_ack = htonl(175); p->tcph->th_seq = htonl(328); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER)); FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT)); FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER); FAIL_IF(RB_EMPTY(&ssn->client.seg_tree)); FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))); FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))))); FAIL_IF(RB_EMPTY(&ssn->server.seg_tree)); FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree))); /* response acking a request */ p->tcph->th_ack = htonl(175); p->tcph->th_seq = htonl(328); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOCLIENT; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server)); FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client)); FAIL_IF(f.alproto != ALPROTO_HTTP1); FAIL_IF(f.alproto_ts != ALPROTO_HTTP1); FAIL_IF(f.alproto_tc != ALPROTO_HTTP1); StreamTcpPruneSession(&f, STREAM_TOSERVER); StreamTcpPruneSession(&f, STREAM_TOCLIENT); /* request acking a response */ p->tcph->th_ack = htonl(328); p->tcph->th_seq = htonl(175); p->tcph->th_flags = TH_ACK; p->flowflags = FLOW_PKT_TOSERVER; p->payload_len = 0; p->payload = NULL; FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1); StreamTcpSessionClear(ssn); StreamTcpUTDeinit(stt.ra_ctx); SCFree(p); PASS; } /** * \test Test to make sure that we sent all the segments from the initial * segments to app layer until we have detected the app layer proto. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest40 (void) { Packet *p = PacketGetFromAlloc(); FAIL_IF_NULL(p); Flow *f = NULL; TCPHdr tcph; TcpSession ssn; memset(&tcph, 0, sizeof (TCPHdr)); ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); StreamTcpInitConfig(true); StreamTcpUTSetupSession(&ssn); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(&tv); FAIL_IF_NULL(ra_ctx); uint8_t httpbuf1[] = "P"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ uint8_t httpbuf3[] = "O"; uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ uint8_t httpbuf4[] = "S"; uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ uint8_t httpbuf5[] = "T \r\n"; uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ SET_ISN(&ssn.server, 9); ssn.server.last_ack = 10; SET_ISN(&ssn.client, 9); ssn.client.isn = 9; f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220); FAIL_IF_NULL(f); f->protoctx = &ssn; f->proto = IPPROTO_TCP; p->flow = f; tcph.th_win = htons(5480); tcph.th_seq = htonl(10); tcph.th_ack = htonl(10); tcph.th_flags = TH_ACK|TH_PUSH; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf1; p->payload_len = httplen1; ssn.state = TCP_ESTABLISHED; TcpStream *s = &ssn.client; SCLogDebug("1 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(10); tcph.th_ack = htonl(11); s = &ssn.server; ssn.server.last_ack = 11; SCLogDebug("2 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf3; p->payload_len = httplen3; tcph.th_seq = htonl(11); tcph.th_ack = htonl(55); s = &ssn.client; ssn.client.last_ack = 55; SCLogDebug("3 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(55); tcph.th_ack = htonl(12); s = &ssn.server; ssn.server.last_ack = 12; SCLogDebug("4 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); /* check is have the segment in the list and flagged or not */ TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree); FAIL_IF_NULL(seg); FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client))); p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf4; p->payload_len = httplen4; tcph.th_seq = htonl(12); tcph.th_ack = htonl(100); s = &ssn.client; ssn.client.last_ack = 100; SCLogDebug("5 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(100); tcph.th_ack = htonl(13); s = &ssn.server; ssn.server.last_ack = 13; SCLogDebug("6 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf5; p->payload_len = httplen5; tcph.th_seq = htonl(13); tcph.th_ack = htonl(145); s = &ssn.client; ssn.client.last_ack = 145; SCLogDebug("7 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(145); tcph.th_ack = htonl(16); s = &ssn.server; ssn.server.last_ack = 16; SCLogDebug("8 -- start"); FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); FAIL_IF(f->alproto != ALPROTO_HTTP1); StreamTcpUTClearSession(&ssn); StreamTcpReassembleFreeThreadCtx(ra_ctx); StreamTcpFreeConfig(true); SCFree(p); UTHFreeFlow(f); PASS; } /** \test Test the memcap incrementing/decrementing and memcap check */ static int StreamTcpReassembleTest44(void) { StreamTcpInitConfig(true); uint32_t memuse = SC_ATOMIC_GET(ra_memuse); StreamTcpReassembleIncrMemuse(500); FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500)); StreamTcpReassembleDecrMemuse(500); FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse); FAIL_IF(StreamTcpReassembleCheckMemcap(500) != 1); FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0); StreamTcpFreeConfig(true); FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0); PASS; } /** * \test Test to make sure that reassembly_depth is enforced. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest45 (void) { TcpReassemblyThreadCtx *ra_ctx = NULL; TcpSession ssn; ThreadVars tv; memset(&tv, 0, sizeof(tv)); uint8_t payload[100] = {0}; uint16_t payload_size = 100; StreamTcpUTInit(&ra_ctx); stream_config.reassembly_depth = 100; StreamTcpUTSetupSession(&ssn); ssn.reassembly_depth = 100; StreamTcpUTSetupStream(&ssn.server, 100); StreamTcpUTSetupStream(&ssn.client, 100); int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size); FAIL_IF(r != 0); FAIL_IF(ssn.client.flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED); r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size); FAIL_IF(r != 0); FAIL_IF(!(ssn.client.flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED)); StreamTcpUTClearStream(&ssn.server); StreamTcpUTClearStream(&ssn.client); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); PASS; } /** * \test Test the unlimited config value of reassembly depth. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest46 (void) { int result = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; TcpSession ssn; ThreadVars tv; memset(&tv, 0, sizeof(tv)); uint8_t payload[100] = {0}; uint16_t payload_size = 100; StreamTcpUTInit(&ra_ctx); stream_config.reassembly_depth = 0; StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.server, 100); StreamTcpUTSetupStream(&ssn.client, 100); int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size); if (r != 0) goto end; if (ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: "); goto end; } r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size); if (r != 0) goto end; if (ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: "); goto end; } result = 1; end: StreamTcpUTClearStream(&ssn.server); StreamTcpUTClearStream(&ssn.client); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return result; } /** * \test Test to make sure we detect the sequence wrap around and continue * stream reassembly properly. * * \retval On success it returns 1 and on failure 0. */ static int StreamTcpReassembleTest47 (void) { Packet *p = PacketGetFromAlloc(); FAIL_IF(unlikely(p == NULL)); Flow *f = NULL; TCPHdr tcph; TcpSession ssn; ThreadVars tv; memset(&tcph, 0, sizeof (TCPHdr)); memset(&tv, 0, sizeof (ThreadVars)); StreamTcpInitConfig(true); StreamTcpUTSetupSession(&ssn); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(&tv); uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ SET_ISN(&ssn.server, 572799781UL); ssn.server.last_ack = 572799782UL; SET_ISN(&ssn.client, 4294967289UL); ssn.client.last_ack = 21; f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220); FAIL_IF(f == NULL); f->protoctx = &ssn; f->proto = IPPROTO_TCP; p->flow = f; tcph.th_win = htons(5480); ssn.state = TCP_ESTABLISHED; TcpStream *s = NULL; uint8_t cnt = 0; for (cnt=0; cnt < httplen1; cnt++) { tcph.th_seq = htonl(ssn.client.isn + 1 + cnt); tcph.th_ack = htonl(572799782UL); tcph.th_flags = TH_ACK|TH_PUSH; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; p->payload = &httpbuf1[cnt]; p->payload_len = 1; s = &ssn.client; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); p->flowflags = FLOW_PKT_TOCLIENT; p->payload = NULL; p->payload_len = 0; tcph.th_seq = htonl(572799782UL); tcph.th_ack = htonl(ssn.client.isn + 1 + cnt); tcph.th_flags = TH_ACK; p->tcph = &tcph; s = &ssn.server; FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1); } FAIL_IF(f->alproto != ALPROTO_HTTP1); StreamTcpUTClearSession(&ssn); StreamTcpReassembleFreeThreadCtx(ra_ctx); StreamTcpFreeConfig(true); SCFree(p); UTHFreeFlow(f); PASS; } /** \test 3 in order segments in inline reassembly */ static int StreamTcpReassembleInlineTest01(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(12); p->flow = &f; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { printf("failed to add segment 1: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 17; ret = 1; end: FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test 3 in order segments, then reassemble, add one more and reassemble again. * test the sliding window reassembly. */ static int StreamTcpReassembleInlineTest02(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(12); p->flow = &f; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { printf("failed to add segment 1: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 17; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { printf("failed to add segment 4: "); goto end; } ssn.client.next_seq = 22; ret = 1; end: FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test 3 in order segments, then reassemble, add one more and reassemble again. * test the sliding window reassembly with a small window size so that we * cutting off at the start (left edge) */ static int StreamTcpReassembleInlineTest03(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); stream_config.reassembly_toserver_chunk_size = 15; uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(12); p->flow = &f; p->flowflags |= FLOW_PKT_TOSERVER; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { printf("failed to add segment 1: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 17; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { printf("failed to add segment 4: "); goto end; } ssn.client.next_seq = 22; p->tcph->th_seq = htonl(17); ret = 1; end: FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test 3 in order segments, then reassemble, add one more and reassemble again. * test the sliding window reassembly with a small window size so that we * cutting off at the start (left edge) with small packet overlap. */ static int StreamTcpReassembleInlineTest04(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); stream_config.reassembly_toserver_chunk_size = 16; uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(12); p->flow = &f; p->flowflags |= FLOW_PKT_TOSERVER; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { printf("failed to add segment 1: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 17; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { printf("failed to add segment 4: "); goto end; } ssn.client.next_seq = 22; p->tcph->th_seq = htonl(17); ret = 1; end: FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test 3 in order segments, then reassemble, add one more and reassemble again. * test the sliding window reassembly with a small window size so that we * cutting off at the start (left edge). Test if the first segment is * removed from the list. */ static int StreamTcpReassembleInlineTest08(void) { TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; memset(&tv, 0x00, sizeof(tv)); TcpSession ssn; Flow f; StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); stream_config.reassembly_toserver_chunk_size = 15; f.protoctx = &ssn; uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); FAIL_IF(p == NULL); p->tcph->th_seq = htonl(12); p->flow = &f; p->flowflags |= FLOW_PKT_TOSERVER; FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1); ssn.client.next_seq = 17; FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1); ssn.client.next_seq = 22; p->tcph->th_seq = htonl(17); StreamTcpPruneSession(&f, STREAM_TOSERVER); TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree); FAIL_IF_NULL(seg); FAIL_IF_NOT(seg->seq == 2); FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); PASS; } /** \test 3 in order segments, then reassemble, add one more and reassemble again. * test the sliding window reassembly with a small window size so that we * cutting off at the start (left edge). Test if the first segment is * removed from the list. */ static int StreamTcpReassembleInlineTest09(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); stream_config.reassembly_toserver_chunk_size = 20; uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(17); p->flow = &f; p->flowflags |= FLOW_PKT_TOSERVER; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { printf("failed to add segment 1: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 12; ssn.client.last_ack = 10; /* close the GAP and see if we properly reassemble and update base_seq */ if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { printf("failed to add segment 4: "); goto end; } ssn.client.next_seq = 22; p->tcph->th_seq = htonl(12); TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree); FAIL_IF_NULL(seg); FAIL_IF_NOT(seg->seq == 2); ret = 1; end: FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test App Layer reassembly. */ static int StreamTcpReassembleInlineTest10(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow *f = NULL; Packet *p = NULL; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.server, 1); ssn.server.last_ack = 2; StreamTcpUTSetupStream(&ssn.client, 1); ssn.client.last_ack = 2; ssn.data_first_seen_dir = STREAM_TOSERVER; f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80); if (f == NULL) goto end; f->protoctx = &ssn; f->proto = IPPROTO_TCP; uint8_t stream_payload1[] = "GE"; uint8_t stream_payload2[] = "T /"; uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n"; p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); if (p == NULL) { printf("couldn't get a packet: "); goto end; } p->tcph->th_seq = htonl(7); p->flow = f; p->flowflags = FLOW_PKT_TOSERVER; if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) { printf("failed to add segment 1: "); goto end; } ssn.client.next_seq = 4; int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET); if (r < 0) { printf("StreamTcpReassembleAppLayer failed: "); goto end; } /* ssn.server.ra_app_base_seq should be isn here. */ if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) { printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq); goto end; } if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) { printf("failed to add segment 2: "); goto end; } if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) { printf("failed to add segment 3: "); goto end; } ssn.client.next_seq = 19; r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET); if (r < 0) { printf("StreamTcpReassembleAppLayer failed: "); goto end; } FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17); ret = 1; end: UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); UTHFreeFlow(f); return ret; } /** \test test insert with overlap */ static int StreamTcpReassembleInsertTest01(void) { TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; Flow f; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); ssn.client.os_policy = OS_POLICY_LAST; FLOW_INITIALIZE(&f); uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); FAIL_IF(p == NULL); p->tcph->th_seq = htonl(12); p->flow = &f; FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1); FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1); ssn.client.next_seq = 21; FLOW_DESTROY(&f); UTHFreePacket(p); StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); PASS; } /** \test test insert with overlaps */ static int StreamTcpReassembleInsertTest02(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); int i; for (i = 2; i < 10; i++) { int len; len = i % 2; if (len == 0) len = 1; int seq; seq = i * 10; if (seq < 2) seq = 2; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) { printf("failed to add segment 1: "); goto end; } } if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) { printf("failed to add segment 2: "); goto end; } ret = 1; end: StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } /** \test test insert with overlaps */ static int StreamTcpReassembleInsertTest03(void) { int ret = 0; TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; TcpSession ssn; memset(&tv, 0x00, sizeof(tv)); StreamTcpUTInit(&ra_ctx); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.client, 1); if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) { printf("failed to add segment 2: "); goto end; } int i; for (i = 2; i < 10; i++) { int len; len = i % 2; if (len == 0) len = 1; int seq; seq = i * 10; if (seq < 2) seq = 2; if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) { printf("failed to add segment 2: "); goto end; } } ret = 1; end: StreamTcpUTClearSession(&ssn); StreamTcpUTDeinit(ra_ctx); return ret; } #include "tests/stream-tcp-reassemble.c" #endif /* UNITTESTS */ /** \brief The Function Register the Unit tests to test the reassembly engine * for various OS policies. */ void StreamTcpReassembleRegisterTests(void) { #ifdef UNITTESTS UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test", StreamTcpReassembleTest25); UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test", StreamTcpReassembleTest26); UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test", StreamTcpReassembleTest27); UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test", StreamTcpReassembleTest28); UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test", StreamTcpReassembleTest29); UtRegisterTest("StreamTcpReassembleTest33 -- Bug test", StreamTcpReassembleTest33); UtRegisterTest("StreamTcpReassembleTest34 -- Bug test", StreamTcpReassembleTest34); UtRegisterTest("StreamTcpReassembleTest39 -- app proto test", StreamTcpReassembleTest39); UtRegisterTest("StreamTcpReassembleTest40 -- app proto test", StreamTcpReassembleTest40); UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test", StreamTcpReassembleTest44); UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test", StreamTcpReassembleTest45); UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test", StreamTcpReassembleTest46); UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test", StreamTcpReassembleTest47); UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra", StreamTcpReassembleInlineTest01); UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2", StreamTcpReassembleInlineTest02); UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3", StreamTcpReassembleInlineTest03); UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4", StreamTcpReassembleInlineTest04); UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup", StreamTcpReassembleInlineTest08); UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup", StreamTcpReassembleInlineTest09); UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10", StreamTcpReassembleInlineTest10); UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap", StreamTcpReassembleInsertTest01); UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap", StreamTcpReassembleInsertTest02); UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap", StreamTcpReassembleInsertTest03); StreamTcpInlineRegisterTests(); StreamTcpUtilRegisterTests(); StreamTcpListRegisterTests(); StreamTcpReassembleRawRegisterTests(); #endif /* UNITTESTS */ }