diff options
Diffstat (limited to 'src/input')
| -rw-r--r-- | src/input/fpcap.c | 2 | ||||
| -rw-r--r-- | src/input/fpcap.h | 2 | ||||
| -rw-r--r-- | src/input/fpcap.hh | 10 | ||||
| -rw-r--r-- | src/input/fpcap.lua | 2 | ||||
| -rw-r--r-- | src/input/mmpcap.c | 2 | ||||
| -rw-r--r-- | src/input/mmpcap.h | 2 | ||||
| -rw-r--r-- | src/input/mmpcap.hh | 10 | ||||
| -rw-r--r-- | src/input/mmpcap.lua | 2 | ||||
| -rw-r--r-- | src/input/pcap.c | 2 | ||||
| -rw-r--r-- | src/input/pcap.h | 2 | ||||
| -rw-r--r-- | src/input/pcap.hh | 10 | ||||
| -rw-r--r-- | src/input/pcap.lua | 2 | ||||
| -rw-r--r-- | src/input/zero.lua | 2 | ||||
| -rw-r--r-- | src/input/zmmpcap.c | 677 | ||||
| -rw-r--r-- | src/input/zmmpcap.h | 31 | ||||
| -rw-r--r-- | src/input/zmmpcap.hh | 77 | ||||
| -rw-r--r-- | src/input/zmmpcap.lua | 149 | ||||
| -rw-r--r-- | src/input/zpcap.c | 159 | ||||
| -rw-r--r-- | src/input/zpcap.h | 2 | ||||
| -rw-r--r-- | src/input/zpcap.hh | 14 | ||||
| -rw-r--r-- | src/input/zpcap.lua | 12 |
21 files changed, 1117 insertions, 54 deletions
diff --git a/src/input/fpcap.c b/src/input/fpcap.c index ec9cfff..04885c6 100644 --- a/src/input/fpcap.c +++ b/src/input/fpcap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/fpcap.h b/src/input/fpcap.h index dfa2dbe..4f2967e 100644 --- a/src/input/fpcap.h +++ b/src/input/fpcap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/fpcap.hh b/src/input/fpcap.hh index 0337503..dd8e19d 100644 --- a/src/input/fpcap.hh +++ b/src/input/fpcap.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. @@ -18,10 +18,10 @@ * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. */ -//lua:require("dnsjit.core.log") -//lua:require("dnsjit.core.receiver_h") -//lua:require("dnsjit.core.producer_h") -//lua:require("dnsjit.core.object.pcap_h") +// lua:require("dnsjit.core.log") +// lua:require("dnsjit.core.receiver_h") +// lua:require("dnsjit.core.producer_h") +// lua:require("dnsjit.core.object.pcap_h") typedef struct input_fpcap { core_log_t _log; diff --git a/src/input/fpcap.lua b/src/input/fpcap.lua index c52c7a4..e0d9447 100644 --- a/src/input/fpcap.lua +++ b/src/input/fpcap.lua @@ -1,4 +1,4 @@ --- Copyright (c) 2018-2022, OARC, Inc. +-- Copyright (c) 2018-2024 OARC, Inc. -- All rights reserved. -- -- This file is part of dnsjit. diff --git a/src/input/mmpcap.c b/src/input/mmpcap.c index ee2baa2..2caf5cf 100644 --- a/src/input/mmpcap.c +++ b/src/input/mmpcap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/mmpcap.h b/src/input/mmpcap.h index e7c66e8..8844cb7 100644 --- a/src/input/mmpcap.h +++ b/src/input/mmpcap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/mmpcap.hh b/src/input/mmpcap.hh index de446b7..387d5a6 100644 --- a/src/input/mmpcap.hh +++ b/src/input/mmpcap.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. @@ -18,10 +18,10 @@ * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. */ -//lua:require("dnsjit.core.log") -//lua:require("dnsjit.core.receiver_h") -//lua:require("dnsjit.core.producer_h") -//lua:require("dnsjit.core.object.pcap_h") +// lua:require("dnsjit.core.log") +// lua:require("dnsjit.core.receiver_h") +// lua:require("dnsjit.core.producer_h") +// lua:require("dnsjit.core.object.pcap_h") typedef struct input_mmpcap { core_log_t _log; diff --git a/src/input/mmpcap.lua b/src/input/mmpcap.lua index f03636e..983d693 100644 --- a/src/input/mmpcap.lua +++ b/src/input/mmpcap.lua @@ -1,4 +1,4 @@ --- Copyright (c) 2018-2022, OARC, Inc. +-- Copyright (c) 2018-2024 OARC, Inc. -- All rights reserved. -- -- This file is part of dnsjit. diff --git a/src/input/pcap.c b/src/input/pcap.c index 5506bac..2304109 100644 --- a/src/input/pcap.c +++ b/src/input/pcap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/pcap.h b/src/input/pcap.h index c6d65ce..5b091ba 100644 --- a/src/input/pcap.h +++ b/src/input/pcap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/pcap.hh b/src/input/pcap.hh index 900338c..de694a7 100644 --- a/src/input/pcap.hh +++ b/src/input/pcap.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. @@ -22,10 +22,10 @@ typedef struct pcap {} pcap_t; #endif -//lua:require("dnsjit.core.log") -//lua:require("dnsjit.core.receiver_h") -//lua:require("dnsjit.core.producer_h") -//lua:require("dnsjit.core.object.pcap_h") +// lua:require("dnsjit.core.log") +// lua:require("dnsjit.core.receiver_h") +// lua:require("dnsjit.core.producer_h") +// lua:require("dnsjit.core.object.pcap_h") typedef struct input_pcap { core_log_t _log; diff --git a/src/input/pcap.lua b/src/input/pcap.lua index a092f85..237a8db 100644 --- a/src/input/pcap.lua +++ b/src/input/pcap.lua @@ -1,4 +1,4 @@ --- Copyright (c) 2018-2022, OARC, Inc. +-- Copyright (c) 2018-2024 OARC, Inc. -- All rights reserved. -- -- This file is part of dnsjit. diff --git a/src/input/zero.lua b/src/input/zero.lua index dd3c6f8..6c50d91 100644 --- a/src/input/zero.lua +++ b/src/input/zero.lua @@ -1,4 +1,4 @@ --- Copyright (c) 2018-2022, OARC, Inc. +-- Copyright (c) 2018-2024 OARC, Inc. -- All rights reserved. -- -- This file is part of dnsjit. diff --git a/src/input/zmmpcap.c b/src/input/zmmpcap.c new file mode 100644 index 0000000..5c920e1 --- /dev/null +++ b/src/input/zmmpcap.c @@ -0,0 +1,677 @@ +/* + * Copyright (c) 2018-2024 OARC, Inc. + * All rights reserved. + * + * This file is part of dnsjit. + * + * dnsjit is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dnsjit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "config.h" + +#include "input/zmmpcap.h" +#include "core/assert.h" +#include "core/object/pcap.h" + +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> +#ifdef HAVE_ENDIAN_H +#include <endian.h> +#else +#ifdef HAVE_SYS_ENDIAN_H +#include <sys/endian.h> +#else +#ifdef HAVE_MACHINE_ENDIAN_H +#include <machine/endian.h> +#endif +#endif +#endif +#ifdef HAVE_BYTESWAP_H +#include <byteswap.h> +#endif +#ifndef bswap_16 +#ifndef bswap16 +#define bswap_16(x) swap16(x) +#define bswap_32(x) swap32(x) +#define bswap_64(x) swap64(x) +#else +#define bswap_16(x) bswap16(x) +#define bswap_32(x) bswap32(x) +#define bswap_64(x) bswap64(x) +#endif +#endif +#include <pcap/pcap.h> + +#ifdef HAVE_LZ4 +#include <lz4frame.h> +struct _lz4_ctx { + LZ4F_dctx* ctx; + LZ4F_decompressOptions_t opts; +}; +#define lz4 ((struct _lz4_ctx*)self->comp_ctx) +#endif + +#ifdef HAVE_ZSTD +#include <zstd.h> +struct _zstd_ctx { + ZSTD_DCtx* ctx; + ZSTD_inBuffer in; + ZSTD_outBuffer out; +}; +#define zstd ((struct _zstd_ctx*)self->comp_ctx) +#endif + +#include <zlib.h> +struct _gzip_ctx { + z_stream strm; +}; +#define gzip ((struct _gzip_ctx*)self->comp_ctx) + +#ifdef HAVE_LZMA +#include <lzma.h> +struct _lzma_ctx { + lzma_stream strm; +}; +static lzma_stream lzma_stream_init = LZMA_STREAM_INIT; +#define lzma ((struct _lzma_ctx*)self->comp_ctx) +#endif + +static core_log_t _log = LOG_T_INIT("input.zmmpcap"); +static input_zmmpcap_t _defaults = { + LOG_T_INIT_OBJ("input.zmmpcap"), + 0, 0, + 0, 0, 0, + CORE_OBJECT_PCAP_INIT(0), + input_zmmpcap_type_none, 0, + 0, 0, 0, 0, + -1, 0, 0, 0, MAP_FAILED, + 0, 0, 0, 0, 0, 0, 0, + 0 +}; + +core_log_t* input_zmmpcap_log() +{ + return &_log; +} + +void input_zmmpcap_init(input_zmmpcap_t* self) +{ + mlassert_self(); + + *self = _defaults; +} + +void input_zmmpcap_destroy(input_zmmpcap_t* self) +{ + mlassert_self(); + + free(self->out); + + switch (self->compression) { +#ifdef HAVE_LZ4 + case input_zmmpcap_type_lz4: { + LZ4F_errorCode_t code; + + if (lz4 && lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { + lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); + } + free(lz4); + break; + } +#endif +#ifdef HAVE_ZSTD + case input_zmmpcap_type_zstd: + if (zstd && zstd->ctx) { + ZSTD_freeDCtx(zstd->ctx); + } + free(zstd); + break; +#endif + case input_zmmpcap_type_gzip: + if (gzip) { + inflateEnd(&gzip->strm); + } + free(gzip); + break; +#ifdef HAVE_LZMA + case input_zmmpcap_type_lzma: + if (lzma) { + lzma_end(&lzma->strm); + } + free(lzma); + break; +#endif + default: + break; + } + + if (self->map != MAP_FAILED) { + munmap(self->map, self->len); + } + if (self->fd > -1) { + close(self->fd); + } + free(self->buf); +} + +static ssize_t _read(input_zmmpcap_t* self, void* dst, size_t len, void** dstp) +{ + switch (self->compression) { +#ifdef HAVE_LZ4 + case input_zmmpcap_type_lz4: { + size_t need = len; + + if (dstp && self->out_have >= need) { + *dstp = self->out + self->out_at; + self->out_have -= need; + self->out_at += need; + return len; + } + + for (;;) { + if (self->out_have >= need) { + memcpy(dst, self->out + self->out_at, need); + self->out_have -= need; + self->out_at += need; + return len; + } + + memcpy(dst, self->out + self->out_at, self->out_have); + need -= self->out_have; + dst += self->out_have; + + if (self->at >= self->len) { + return 0; + } + + size_t dst_size = self->out_size, src_size = self->len - self->at; + size_t code = LZ4F_decompress(lz4->ctx, self->out, &dst_size, &self->map[self->at], &src_size, &lz4->opts); + if (LZ4F_isError(code)) { + lfatal("LZ4F_decompress() failed: %s", LZ4F_getErrorName(code)); + } + + self->at += src_size; + self->out_at = 0; + self->out_have = dst_size; + } + } +#endif +#ifdef HAVE_ZSTD + case input_zmmpcap_type_zstd: { + size_t need = len; + + if (dstp && self->out_have >= need) { + *dstp = self->out + self->out_at; + self->out_have -= need; + self->out_at += need; + return len; + } + + for (;;) { + if (self->out_have >= need) { + memcpy(dst, self->out + self->out_at, need); + self->out_have -= need; + self->out_at += need; + return len; + } + + memcpy(dst, self->out + self->out_at, self->out_have); + need -= self->out_have; + dst += self->out_have; + + if (zstd->in.pos >= zstd->in.size) { + return 0; + } + + zstd->out.size = self->out_size; + zstd->out.pos = 0; + size_t code = ZSTD_decompressStream(zstd->ctx, &zstd->out, &zstd->in); + if (ZSTD_isError(code)) { + lfatal("ZSTD_decompressStream() failed: %s", ZSTD_getErrorName(code)); + } + + self->out_have = zstd->out.pos; + self->out_at = 0; + } + } +#endif + case input_zmmpcap_type_gzip: { + size_t need = len; + + if (dstp && self->out_have >= need) { + *dstp = self->out + self->out_at; + self->out_have -= need; + self->out_at += need; + return len; + } + + for (;;) { + if (self->out_have >= need) { + memcpy(dst, self->out + self->out_at, need); + self->out_have -= need; + self->out_at += need; + return len; + } + + memcpy(dst, self->out + self->out_at, self->out_have); + need -= self->out_have; + dst += self->out_have; + + if (gzip->strm.avail_in <= 0) { + return 0; + } + + gzip->strm.next_out = self->out; + gzip->strm.avail_out = self->out_size; + + self->out_at = 0; + int ret = inflate(&gzip->strm, Z_NO_FLUSH); + if (ret != Z_OK) { + if (ret == Z_STREAM_END) { + self->out_have = self->out_size - gzip->strm.avail_out; + if (self->out_have > 0) { + continue; + } + return 0; + } + lfatal("inflate() failed: %d: %s", ret, gzip->strm.msg); + } + + self->out_have = self->out_size - gzip->strm.avail_out; + } + } +#ifdef HAVE_LZMA + case input_zmmpcap_type_lzma: { + size_t need = len; + + if (dstp && self->out_have >= need) { + *dstp = self->out + self->out_at; + self->out_have -= need; + self->out_at += need; + return len; + } + + lzma_action action = LZMA_RUN; + for (;;) { + if (self->out_have >= need) { + memcpy(dst, self->out + self->out_at, need); + self->out_have -= need; + self->out_at += need; + return len; + } + + memcpy(dst, self->out + self->out_at, self->out_have); + need -= self->out_have; + dst += self->out_have; + + if (lzma->strm.avail_in <= 0) { + return 0; + } + + lzma->strm.next_out = self->out; + lzma->strm.avail_out = self->out_size; + + self->out_at = 0; + lzma_ret ret = lzma_code(&lzma->strm, action); + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + self->out_have = self->out_size - lzma->strm.avail_out; + if (self->out_have > 0) { + continue; + } + return 0; + } + lfatal("lzma_code() failed: %d", ret); + } + + self->out_have = self->out_size - lzma->strm.avail_out; + } + } +#endif + default: + return 0; + } +} + +int input_zmmpcap_open(input_zmmpcap_t* self, const char* file) +{ + struct stat sb; + mlassert_self(); + lassert(file, "file is nil"); + + if (self->fd != -1) { + lfatal("already opened"); + } + + if ((self->fd = open(file, O_RDONLY)) < 0) { + lcritical("open(%s) error %s", file, core_log_errstr(errno)); + return -1; + } + + if (fstat(self->fd, &sb)) { + lcritical("stat(%s) error %s", file, core_log_errstr(errno)); + return -1; + } + self->len = sb.st_size; + + if ((self->map = mmap(0, self->len, PROT_READ, MAP_PRIVATE, self->fd, 0)) == MAP_FAILED) { + lcritical("mmap(%s) error %s", file, core_log_errstr(errno)); + return -1; + } + + switch (self->compression) { +#ifdef HAVE_LZ4 + case input_zmmpcap_type_lz4: { + LZ4F_errorCode_t code; + + if (lz4 && lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { + lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); + } + free(lz4); + free(self->out); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _lz4_ctx))); + if ((code = LZ4F_createDecompressionContext(&lz4->ctx, LZ4F_VERSION))) { + lfatal("LZ4F_createDecompressionContext() failed: %s", LZ4F_getErrorName(code)); + } + lz4->opts.stableDst = 1; + + self->out_size = 256 * 1024; + lfatal_oom(self->out = malloc(self->out_size)); + break; + } +#endif +#ifdef HAVE_ZSTD + case input_zmmpcap_type_zstd: + if (zstd && zstd->ctx) { + ZSTD_freeDCtx(zstd->ctx); + } + free(zstd); + free(self->out); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _zstd_ctx))); + lfatal_oom(zstd->ctx = ZSTD_createDCtx()); + self->out_size = ZSTD_DStreamOutSize(); + lfatal_oom(self->out = malloc(self->out_size + 1)); + + zstd->in.src = self->map; + zstd->in.size = self->len; + zstd->out.dst = self->out; + zstd->out.size = self->out_size; + break; +#endif + case input_zmmpcap_type_gzip: { + if (gzip) { + inflateEnd(&gzip->strm); + } + free(gzip); + free(self->out); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _gzip_ctx))); + int ret = inflateInit2(&gzip->strm, 32); + if (ret != Z_OK) { + lcritical("inflateInit() error: %d", ret); + return -1; + } + + self->out_size = 256 * 1024; + lfatal_oom(self->out = malloc(self->out_size)); + + gzip->strm.next_in = self->map; + gzip->strm.avail_in = self->len; + break; + } +#ifdef HAVE_LZMA + case input_zmmpcap_type_lzma: { + if (lzma) { + lzma_end(&lzma->strm); + } + free(lzma); + free(self->out); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _lzma_ctx))); + lzma->strm = lzma_stream_init; + lzma_ret ret = lzma_stream_decoder(&lzma->strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) { + lcritical("lzma_stream_decoder() error: %d", ret); + return -1; + } + + self->out_size = 256 * 1024; + lfatal_oom(self->out = malloc(self->out_size)); + + lzma->strm.next_in = self->map; + lzma->strm.avail_in = self->len; + break; + } +#endif + default: + lcritical("no support for selected compression"); + return -2; + } + + if (_read(self, &self->magic_number, 4, 0) != 4 + || _read(self, &self->version_major, 2, 0) != 2 + || _read(self, &self->version_minor, 2, 0) != 2 + || _read(self, &self->thiszone, 4, 0) != 4 + || _read(self, &self->sigfigs, 4, 0) != 4 + || _read(self, &self->snaplen, 4, 0) != 4 + || _read(self, &self->network, 4, 0) != 4) { + lcritical("could not read full PCAP header"); + return -2; + } + switch (self->magic_number) { + case 0x4d3cb2a1: + self->is_nanosec = 1; + case 0xd4c3b2a1: + self->is_swapped = 1; + self->version_major = bswap_16(self->version_major); + self->version_minor = bswap_16(self->version_minor); + self->thiszone = (int32_t)bswap_32((uint32_t)self->thiszone); + self->sigfigs = bswap_32(self->sigfigs); + self->snaplen = bswap_32(self->snaplen); + self->network = bswap_32(self->network); + break; + case 0xa1b2c3d4: + case 0xa1b23c4d: + break; + default: + lcritical("invalid PCAP header"); + return -2; + } + + if (self->version_major != 2 || self->version_minor != 4) { + lcritical("unsupported PCAP version v%u.%u", self->version_major, self->version_minor); + return -2; + } + + /* + * Translation taken from https://github.com/the-tcpdump-group/libpcap/blob/90543970fd5fbed261d3637f5ec4811d7dde4e49/pcap-common.c#L1212 . + */ + switch (self->network) { + case 101: /* LINKTYPE_RAW */ + self->linktype = DLT_RAW; + break; +#ifdef DLT_FR + case 107: /* LINKTYPE_FRELAY */ + self->linktype = DLT_FR; + break; +#endif + case 100: /* LINKTYPE_ATM_RFC1483 */ + self->linktype = DLT_ATM_RFC1483; + break; + case 102: /* LINKTYPE_SLIP_BSDOS */ + self->linktype = DLT_SLIP_BSDOS; + break; + case 103: /* LINKTYPE_PPP_BSDOS */ + self->linktype = DLT_PPP_BSDOS; + break; + case 104: /* LINKTYPE_C_HDLC */ + self->linktype = DLT_C_HDLC; + break; + case 106: /* LINKTYPE_ATM_CLIP */ + self->linktype = DLT_ATM_CLIP; + break; + case 50: /* LINKTYPE_PPP_HDLC */ + self->linktype = DLT_PPP_SERIAL; + break; + case 51: /* LINKTYPE_PPP_ETHER */ + self->linktype = DLT_PPP_ETHER; + break; + default: + self->linktype = self->network; + } + + free(self->buf); + lfatal_oom(self->buf = malloc(self->snaplen)); + self->prod_pkt.snaplen = self->snaplen; + self->prod_pkt.linktype = self->linktype; + self->prod_pkt.is_swapped = self->is_swapped; + + ldebug("pcap v%u.%u snaplen:%lu %s", self->version_major, self->version_minor, self->snaplen, self->is_swapped ? " swapped" : ""); + + return 0; +} + +int input_zmmpcap_run(input_zmmpcap_t* self) +{ + struct { + uint32_t ts_sec; + uint32_t ts_usec; + uint32_t incl_len; + uint32_t orig_len; + } hdr; + int ret; + core_object_pcap_t pkt = CORE_OBJECT_PCAP_INIT(0); + mlassert_self(); + + if (self->map == MAP_FAILED) { + lfatal("no PCAP opened"); + } + if (!self->recv) { + lfatal("no receiver set"); + } + + pkt.snaplen = self->snaplen; + pkt.linktype = self->linktype; + pkt.is_swapped = self->is_swapped; + + while ((ret = _read(self, &hdr, 16, 0)) == 16) { + if (self->is_swapped) { + hdr.ts_sec = bswap_32(hdr.ts_sec); + hdr.ts_usec = bswap_32(hdr.ts_usec); + hdr.incl_len = bswap_32(hdr.incl_len); + hdr.orig_len = bswap_32(hdr.orig_len); + } + if (hdr.incl_len > self->snaplen) { + lwarning("invalid packet length, larger then snaplen"); + return -1; + } + pkt.bytes = (unsigned char*)self->buf; + if (_read(self, self->buf, hdr.incl_len, (void**)&pkt.bytes) != hdr.incl_len) { + lwarning("could not read all of packet, aborting"); + return -1; + } + + self->pkts++; + + pkt.ts.sec = hdr.ts_sec; + if (self->is_nanosec) { + pkt.ts.nsec = hdr.ts_usec; + } else { + pkt.ts.nsec = hdr.ts_usec * 1000; + } + pkt.caplen = hdr.incl_len; + pkt.len = hdr.orig_len; + + self->recv(self->ctx, (core_object_t*)&pkt); + } + if (ret) { + lwarning("could not read next PCAP header, aborting"); + return -1; + } + + return 0; +} + +static const core_object_t* _produce(input_zmmpcap_t* self) +{ + struct { + uint32_t ts_sec; + uint32_t ts_usec; + uint32_t incl_len; + uint32_t orig_len; + } hdr; + int ret; + mlassert_self(); + + if (self->is_broken) { + lwarning("PCAP is broken, will not read next packet"); + return 0; + } + + if ((ret = _read(self, &hdr, 16, 0)) != 16) { + if (ret) { + lwarning("could not read next PCAP header, aborting"); + self->is_broken = 1; + } + return 0; + } + + if (self->is_swapped) { + hdr.ts_sec = bswap_32(hdr.ts_sec); + hdr.ts_usec = bswap_32(hdr.ts_usec); + hdr.incl_len = bswap_32(hdr.incl_len); + hdr.orig_len = bswap_32(hdr.orig_len); + } + if (hdr.incl_len > self->snaplen) { + lwarning("invalid packet length, larger then snaplen"); + self->is_broken = 1; + return 0; + } + self->prod_pkt.bytes = (unsigned char*)self->buf; + if (_read(self, self->buf, hdr.incl_len, (void**)&self->prod_pkt.bytes) != hdr.incl_len) { + lwarning("could not read all of packet, aborting"); + self->is_broken = 1; + return 0; + } + + self->pkts++; + + self->prod_pkt.ts.sec = hdr.ts_sec; + if (self->is_nanosec) { + self->prod_pkt.ts.nsec = hdr.ts_usec; + } else { + self->prod_pkt.ts.nsec = hdr.ts_usec * 1000; + } + self->prod_pkt.caplen = hdr.incl_len; + self->prod_pkt.len = hdr.orig_len; + + return (core_object_t*)&self->prod_pkt; +} + +core_producer_t input_zmmpcap_producer(input_zmmpcap_t* self) +{ + mlassert_self(); + + if (self->map == MAP_FAILED) { + lfatal("no PCAP opened"); + } + + return (core_producer_t)_produce; +} diff --git a/src/input/zmmpcap.h b/src/input/zmmpcap.h new file mode 100644 index 0000000..baea3f5 --- /dev/null +++ b/src/input/zmmpcap.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018-2024 OARC, Inc. + * All rights reserved. + * + * This file is part of dnsjit. + * + * dnsjit is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dnsjit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <dnsjit/core/log.h> +#include <dnsjit/core/receiver.h> +#include <dnsjit/core/producer.h> +#include <dnsjit/core/object/pcap.h> + +#ifndef __dnsjit_input_zmmpcap_h +#define __dnsjit_input_zmmpcap_h + +#include <dnsjit/input/zmmpcap.hh> + +#endif diff --git a/src/input/zmmpcap.hh b/src/input/zmmpcap.hh new file mode 100644 index 0000000..7e930cf --- /dev/null +++ b/src/input/zmmpcap.hh @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018-2024 OARC, Inc. + * All rights reserved. + * + * This file is part of dnsjit. + * + * dnsjit is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dnsjit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. + */ + +// lua:require("dnsjit.core.log") +// lua:require("dnsjit.core.receiver_h") +// lua:require("dnsjit.core.producer_h") +// lua:require("dnsjit.core.object.pcap_h") + +typedef enum input_zmmpcap_type { + input_zmmpcap_type_none, + input_zmmpcap_type_lz4, + input_zmmpcap_type_zstd, + input_zmmpcap_type_gzip, + input_zmmpcap_type_lzma +} input_zmmpcap_type_t; + +typedef struct input_zmmpcap { + core_log_t _log; + core_receiver_t recv; + void* ctx; + + uint8_t is_swapped; + uint8_t is_nanosec; + uint8_t is_broken; + + core_object_pcap_t prod_pkt; + + input_zmmpcap_type_t compression; + void* comp_ctx; + + void* out; + size_t out_size; + size_t out_have; + size_t out_at; + + int fd; + size_t len, at; + size_t pkts; + uint8_t *map, *buf; + + uint32_t magic_number; + uint16_t version_major; + uint16_t version_minor; + int32_t thiszone; + uint32_t sigfigs; + uint32_t snaplen; + uint32_t network; + + uint32_t linktype; +} input_zmmpcap_t; + +core_log_t* input_zmmpcap_log(); + +void input_zmmpcap_init(input_zmmpcap_t* self); +void input_zmmpcap_destroy(input_zmmpcap_t* self); +int input_zmmpcap_open(input_zmmpcap_t* self, const char* file); +int input_zmmpcap_run(input_zmmpcap_t* self); +int input_zmmpcap_have_support(input_zmmpcap_t* self); + +core_producer_t input_zmmpcap_producer(input_zmmpcap_t* self); diff --git a/src/input/zmmpcap.lua b/src/input/zmmpcap.lua new file mode 100644 index 0000000..7646869 --- /dev/null +++ b/src/input/zmmpcap.lua @@ -0,0 +1,149 @@ +-- Copyright (c) 2018-2024 OARC, Inc. +-- All rights reserved. +-- +-- This file is part of dnsjit. +-- +-- dnsjit is free software: you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- dnsjit is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with dnsjit. If not, see <http://www.gnu.org/licenses/>. + +-- dnsjit.input.zmmpcap +-- Read input from a PCAP file using mmap() +-- local input = require("dnsjit.input.zmmpcap").new() +-- input:zstd() +-- input:open("file.pcap.zst") +-- input:receiver(filter_or_output) +-- input:run() +-- +-- Read input from a PCAP file by mapping the whole file to memory using +-- .B mmap() +-- and parse the PCAP without libpcap. +-- After opening a file and reading the PCAP header, the attributes are +-- populated. +-- .SS Attributes +-- .TP +-- is_swapped +-- Indicate if the byte order in the PCAP is in reverse order of the host. +-- .TP +-- is_nanosec +-- Indicate if the time stamps are in nanoseconds or not. +-- .TP +-- magic_number +-- Magic number. +-- .TP +-- version_major +-- Major version number. +-- .TP +-- version_minor +-- Minor version number. +-- .TP +-- thiszone +-- GMT to local correction. +-- .TP +-- sigfigs +-- Accuracy of timestamps. +-- .TP +-- snaplen +-- Max length of captured packets, in octets. +-- .TP +-- network +-- The link type found in the PCAP header, see https://www.tcpdump.org/linktypes.html . +-- .TP +-- linktype +-- The data link type, mapped from +-- .IR network . +module(...,package.seeall) + +require("dnsjit.input.zmmpcap_h") +local ffi = require("ffi") +local C = ffi.C + +local t_name = "input_zmmpcap_t" +local input_zmmpcap_t = ffi.typeof(t_name) +local Zmmpcap = {} + +-- Create a new Zmmpcap input. +function Zmmpcap.new() + local self = { + _receiver = nil, + obj = input_zmmpcap_t(), + } + C.input_zmmpcap_init(self.obj) + ffi.gc(self.obj, C.input_zmmpcap_destroy) + return setmetatable(self, { __index = Zmmpcap }) +end + +-- Return the Log object to control logging of this instance or module. +function Zmmpcap:log() + if self == nil then + return C.input_zmmpcap_log() + end + return self.obj._log +end + +-- Set the receiver to pass objects to. +function Zmmpcap:receiver(o) + self.obj.recv, self.obj.ctx = o:receive() + self._receiver = o +end + +-- Return the C functions and context for producing objects. +function Zmmpcap:produce() + return C.input_zmmpcap_producer(self.obj), self.obj +end + +-- Use liblz4 to decompress the input file/data. +function Zmmpcap:lz4() + self.obj.compression = "input_zmmpcap_type_lz4" +end + +-- Use libzstd to decompress the input file/data. +function Zmmpcap:zstd() + self.obj.compression = "input_zmmpcap_type_zstd" +end + +-- Use zlib/gzip to decompress the input file/data. +function Zmmpcap:gzip() + self.obj.compression = "input_zmmpcap_type_gzip" +end + +-- Use liblzma/xz to decompress the input file/data. +function Zmmpcap:lzma() + self.obj.compression = "input_zmmpcap_type_lzma" +end + +-- Return true if support for selected compression library is built in. +function Zmmpcap:have_support() + if C.input_zmmpcap_have_support(self.obj) == 1 then + return true + end + return false +end + +-- Open a PCAP file for processing and read the PCAP header. +-- Returns 0 on success. +function Zmmpcap:open(file) + return C.input_zmmpcap_open(self.obj, file) +end + +-- Start processing packets and send each packet read to the receiver. +-- Returns 0 if all packets was read successfully. +function Zmmpcap:run() + return C.input_zmmpcap_run(self.obj) +end + +-- Return the number of packets seen. +function Zmmpcap:packets() + return tonumber(self.obj.pkts) +end + +return Zmmpcap diff --git a/src/input/zpcap.c b/src/input/zpcap.c index e1d9710..dfeb89b 100644 --- a/src/input/zpcap.c +++ b/src/input/zpcap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. @@ -53,6 +53,7 @@ #endif #include <pcap/pcap.h> #include <string.h> +#include <unistd.h> #ifdef HAVE_LZ4 #include <lz4frame.h> @@ -62,6 +63,7 @@ struct _lz4_ctx { }; #define lz4 ((struct _lz4_ctx*)self->comp_ctx) #endif + #ifdef HAVE_ZSTD #include <zstd.h> struct _zstd_ctx { @@ -72,6 +74,21 @@ struct _zstd_ctx { #define zstd ((struct _zstd_ctx*)self->comp_ctx) #endif +#include <zlib.h> +struct _gzip_ctx { + gzFile fp; +}; +#define gzip ((struct _gzip_ctx*)self->comp_ctx) + +#ifdef HAVE_LZMA +#include <lzma.h> +struct _lzma_ctx { + lzma_stream strm; +}; +static lzma_stream lzma_stream_init = LZMA_STREAM_INIT; +#define lzma ((struct _lzma_ctx*)self->comp_ctx) +#endif + #define MAX_SNAPLEN 0x40000 static core_log_t _log = LOG_T_INIT("input.zpcap"); @@ -108,10 +125,8 @@ void input_zpcap_destroy(input_zpcap_t* self) case input_zpcap_type_lz4: { LZ4F_errorCode_t code; - if (lz4) { - if (lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { - lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); - } + if (lz4 && lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { + lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); } free(lz4); free(self->in); @@ -121,16 +136,29 @@ void input_zpcap_destroy(input_zpcap_t* self) #endif #ifdef HAVE_ZSTD case input_zpcap_type_zstd: - if (zstd) { - if (zstd->ctx) { - ZSTD_freeDCtx(zstd->ctx); - } + if (zstd && zstd->ctx) { + ZSTD_freeDCtx(zstd->ctx); } free(zstd); free(self->in); free(self->out); break; #endif + case input_zpcap_type_gzip: + if (gzip && gzip->fp) { + gzclose(gzip->fp); + } + free(gzip); + break; +#ifdef HAVE_LZMA + case input_zpcap_type_lzma: + if (lzma) { + lzma_end(&lzma->strm); + } + free(lzma); + free(self->out); + break; +#endif default: break; } @@ -148,7 +176,7 @@ static ssize_t _read(input_zpcap_t* self, void* dst, size_t len, void** dstp) case input_zpcap_type_lz4: { size_t need = len; - if (dstp && self->out_have > need) { + if (dstp && self->out_have >= need) { *dstp = self->out + self->out_at; self->out_have -= need; self->out_at += need; @@ -156,7 +184,7 @@ static ssize_t _read(input_zpcap_t* self, void* dst, size_t len, void** dstp) } for (;;) { - if (self->out_have > need) { + if (self->out_have >= need) { memcpy(dst, self->out + self->out_at, need); self->out_have -= need; self->out_at += need; @@ -201,7 +229,7 @@ static ssize_t _read(input_zpcap_t* self, void* dst, size_t len, void** dstp) case input_zpcap_type_zstd: { size_t need = len; - if (dstp && self->out_have > need) { + if (dstp && self->out_have >= need) { *dstp = self->out + self->out_at; self->out_have -= need; self->out_at += need; @@ -209,7 +237,7 @@ static ssize_t _read(input_zpcap_t* self, void* dst, size_t len, void** dstp) } for (;;) { - if (self->out_have > need) { + if (self->out_have >= need) { memcpy(dst, self->out + self->out_at, need); self->out_have -= need; self->out_at += need; @@ -241,6 +269,59 @@ static ssize_t _read(input_zpcap_t* self, void* dst, size_t len, void** dstp) } } #endif + case input_zpcap_type_gzip: + return gzfread(dst, 1, len, gzip->fp); +#ifdef HAVE_LZMA + case input_zpcap_type_lzma: { + size_t need = len; + + if (dstp && self->out_have >= need) { + *dstp = self->out + self->out_at; + self->out_have -= need; + self->out_at += need; + return len; + } + + lzma_action action = LZMA_RUN; + uint8_t inbuf[BUFSIZ]; + for (;;) { + if (self->out_have >= need) { + memcpy(dst, self->out + self->out_at, need); + self->out_have -= need; + self->out_at += need; + return len; + } + + memcpy(dst, self->out + self->out_at, self->out_have); + need -= self->out_have; + dst += self->out_have; + + ssize_t n = fread(inbuf, 1, sizeof(inbuf), self->file); + if (n < 0) { + return n; + } + if (!n) { + action = LZMA_FINISH; + } + + lzma->strm.next_in = inbuf; + lzma->strm.avail_in = n; + lzma->strm.next_out = self->out; + lzma->strm.avail_out = self->out_size; + + lzma_ret ret = lzma_code(&lzma->strm, action); + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + return 0; + } + lfatal("lzma_code() failed: %d", ret); + } + + self->out_at = 0; + self->out_have = self->out_size - lzma->strm.avail_out; + } + } +#endif default: return 0; } @@ -265,10 +346,8 @@ static int _open(input_zpcap_t* self) case input_zpcap_type_lz4: { LZ4F_errorCode_t code; - if (lz4) { - if (lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { - lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); - } + if (lz4 && lz4->ctx && (code = LZ4F_freeDecompressionContext(lz4->ctx))) { + lfatal("LZ4F_freeDecompressionContext() failed: %s", LZ4F_getErrorName(code)); } free(lz4); free(self->in); @@ -290,10 +369,8 @@ static int _open(input_zpcap_t* self) #endif #ifdef HAVE_ZSTD case input_zpcap_type_zstd: - if (zstd) { - if (zstd->ctx) { - ZSTD_freeDCtx(zstd->ctx); - } + if (zstd && zstd->ctx) { + ZSTD_freeDCtx(zstd->ctx); } free(zstd); free(self->in); @@ -311,6 +388,39 @@ static int _open(input_zpcap_t* self) zstd->out.size = self->out_size; break; #endif + case input_zpcap_type_gzip: + free(gzip); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _gzip_ctx))); + + int fd = dup(fileno((FILE*)self->file)); + if (!(gzip->fp = gzdopen(fd, "rb"))) { + lcritical("gzdopen(%d) error: %s", fd, core_log_errstr(errno)); + close(fd); + return -1; + } + break; +#ifdef HAVE_LZMA + case input_zpcap_type_lzma: + if (lzma) { + lzma_end(&lzma->strm); + } + free(lzma); + free(self->out); + + lfatal_oom(self->comp_ctx = calloc(1, sizeof(struct _lzma_ctx))); + lzma->strm = lzma_stream_init; + lzma_ret ret = lzma_stream_decoder(&lzma->strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) { + lcritical("lzma_stream_decoder() error: %d", ret); + return -1; + } + + self->out_size = 256 * 1024; + lfatal_oom(self->out = malloc(self->out_size)); + + break; +#endif default: lcritical("no support for selected compression"); return -2; @@ -393,6 +503,7 @@ static int _open(input_zpcap_t* self) self->linktype = self->network; } + free(self->buf); lfatal_oom(self->buf = malloc(self->snaplen)); self->prod_pkt.snaplen = self->snaplen; self->prod_pkt.linktype = self->linktype; @@ -508,6 +619,12 @@ int input_zpcap_have_support(input_zpcap_t* self) case input_zpcap_type_zstd: return 1; #endif + case input_zpcap_type_gzip: + return 1; +#ifdef HAVE_LZMA + case input_zpcap_type_lzma: + return 1; +#endif default: break; } diff --git a/src/input/zpcap.h b/src/input/zpcap.h index d2c195e..cab85bb 100644 --- a/src/input/zpcap.h +++ b/src/input/zpcap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. diff --git a/src/input/zpcap.hh b/src/input/zpcap.hh index e0079d5..596fc2b 100644 --- a/src/input/zpcap.hh +++ b/src/input/zpcap.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, OARC, Inc. + * Copyright (c) 2018-2024 OARC, Inc. * All rights reserved. * * This file is part of dnsjit. @@ -18,15 +18,17 @@ * along with dnsjit. If not, see <http://www.gnu.org/licenses/>. */ -//lua:require("dnsjit.core.log") -//lua:require("dnsjit.core.receiver_h") -//lua:require("dnsjit.core.producer_h") -//lua:require("dnsjit.core.object.pcap_h") +// lua:require("dnsjit.core.log") +// lua:require("dnsjit.core.receiver_h") +// lua:require("dnsjit.core.producer_h") +// lua:require("dnsjit.core.object.pcap_h") typedef enum input_zpcap_type { input_zpcap_type_none, input_zpcap_type_lz4, - input_zpcap_type_zstd + input_zpcap_type_zstd, + input_zpcap_type_gzip, + input_zpcap_type_lzma } input_zpcap_type_t; typedef struct input_zpcap { diff --git a/src/input/zpcap.lua b/src/input/zpcap.lua index d860a64..5f2869c 100644 --- a/src/input/zpcap.lua +++ b/src/input/zpcap.lua @@ -1,4 +1,4 @@ --- Copyright (c) 2018-2022, OARC, Inc. +-- Copyright (c) 2018-2024 OARC, Inc. -- All rights reserved. -- -- This file is part of dnsjit. @@ -119,6 +119,16 @@ function Zpcap:zstd() self.obj.compression = "input_zpcap_type_zstd" end +-- Use zlib/gzip to decompress the input file/data. +function Zpcap:gzip() + self.obj.compression = "input_zpcap_type_gzip" +end + +-- Use liblzma/xz to decompress the input file/data. +function Zpcap:lzma() + self.obj.compression = "input_zpcap_type_lzma" +end + -- Return true if support for selected compression library is built in. function Zpcap:have_support() if C.input_zpcap_have_support(self.obj) == 1 then |