Adding upstream version 1:7.0.3.upstream/1%7.0.3

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 523 insertions, 0 deletions

diff --git a/src/output-filestore.c b/src/output-filestore.c
new file mode 100644
index 0000000..dcf4c1a
--- /dev/null
+++ b/src/output-filestore.c
@@ -0,0 +1,523 @@
+/* Copyright (C) 2018-2022 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+#include "suricata-common.h"
+#include "output-filestore.h"
+
+#include "stream-tcp.h"
+
+#include "feature.h"
+
+#include "output.h"
+#include "output-json-file.h"
+
+#include "util-conf.h"
+#include "util-misc.h"
+#include "util-path.h"
+#include "util-print.h"
+
+#define MODULE_NAME "OutputFilestore"
+
+/* Create a filestore specific PATH_MAX that is less than the system
+ * PATH_MAX to prevent newer gcc truncation warnings with snprint. */
+#define SHA256_STRING_LEN    (SC_SHA256_LEN * 2)
+#define LEAF_DIR_MAX_LEN 4
+#define FILESTORE_PREFIX_MAX (PATH_MAX - SHA256_STRING_LEN - LEAF_DIR_MAX_LEN)
+
+/* The default log directory, relative to the default log
+ * directory. */
+static const char *default_log_dir = "filestore";
+
+/* Atomic counter of simultaneously open files. */
+static SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt);
+
+typedef struct OutputFilestoreCtx_ {
+    char prefix[FILESTORE_PREFIX_MAX];
+    char tmpdir[FILESTORE_PREFIX_MAX];
+    bool fileinfo;
+    HttpXFFCfg *xff_cfg;
+} OutputFilestoreCtx;
+
+typedef struct OutputFilestoreLogThread_ {
+    OutputFilestoreCtx *ctx;
+    uint16_t counter_max_hits;
+    uint16_t fs_error_counter;
+} OutputFilestoreLogThread;
+
+enum WarnOnceTypes {
+    WOT_OPEN,
+    WOT_WRITE,
+    WOT_UNLINK,
+    WOT_RENAME,
+    WOT_SNPRINTF,
+
+    WOT_MAX,
+};
+
+/* For WARN_ONCE, a record of warnings that have already been
+ * issued. */
+static thread_local bool once_errs[WOT_MAX];
+
+#define WARN_ONCE(wot_type, ...)                                                                   \
+    do {                                                                                           \
+        if (!once_errs[wot_type]) {                                                                \
+            once_errs[wot_type] = true;                                                            \
+            SCLogWarning(__VA_ARGS__);                                                             \
+        }                                                                                          \
+    } while (0)
+
+static uint64_t OutputFilestoreOpenFilesCounter(void)
+{
+    return SC_ATOMIC_GET(filestore_open_file_cnt);
+}
+
+static uint32_t g_file_store_max_open_files = 0;
+
+static void FileSetMaxOpenFiles(uint32_t count)
+{
+    g_file_store_max_open_files = count;
+}
+
+static uint32_t FileGetMaxOpenFiles(void)
+{
+    return g_file_store_max_open_files;
+}
+
+/**
+ * \brief Update the timestamps on a file to match those of another
+ *     file.
+ *
+ * \param src_filename Filename to use as timestamp source.
+ * \param filename Filename to apply timestamps to.
+ */
+static void OutputFilestoreUpdateFileTime(const char *src_filename,
+        const char *filename)
+{
+    struct stat sb;
+    if (stat(src_filename, &sb) != 0) {
+        SCLogDebug("Failed to stat %s: %s", filename, strerror(errno));
+        return;
+    }
+    struct utimbuf utimbuf = {
+        .actime = sb.st_atime,
+        .modtime = sb.st_mtime,
+    };
+    if (utime(filename, &utimbuf) != 0) {
+        SCLogDebug("Failed to update file timestamps: %s: %s", filename,
+                strerror(errno));
+    }
+}
+
+static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLogThread *oft,
+        const OutputFilestoreCtx *ctx, const Packet *p, File *ff, void *tx, const uint64_t tx_id,
+        uint8_t dir)
+{
+    /* Stringify the SHA256 which will be used in the final
+     * filename. */
+    char sha256string[(SC_SHA256_LEN * 2) + 1];
+    PrintHexString(sha256string, sizeof(sha256string), ff->sha256,
+            sizeof(ff->sha256));
+
+    char tmp_filename[PATH_MAX] = "";
+    snprintf(tmp_filename, sizeof(tmp_filename), "%s/file.%u", ctx->tmpdir,
+            ff->file_store_id);
+
+    char final_filename[PATH_MAX] = "";
+    snprintf(final_filename, sizeof(final_filename), "%s/%c%c/%s",
+            ctx->prefix, sha256string[0], sha256string[1], sha256string);
+
+    if (SCPathExists(final_filename)) {
+        OutputFilestoreUpdateFileTime(tmp_filename, final_filename);
+        if (unlink(tmp_filename) != 0) {
+            StatsIncr(tv, oft->fs_error_counter);
+            WARN_ONCE(WOT_UNLINK, "Failed to remove temporary file %s: %s", tmp_filename,
+                    strerror(errno));
+        }
+    } else if (rename(tmp_filename, final_filename) != 0) {
+        StatsIncr(tv, oft->fs_error_counter);
+        WARN_ONCE(WOT_RENAME, "Failed to rename %s to %s: %s", tmp_filename, final_filename,
+                strerror(errno));
+        if (unlink(tmp_filename) != 0) {
+            /* Just increment, don't log as has_fs_errors would
+             * already be set above. */
+            StatsIncr(tv, oft->fs_error_counter);
+        }
+        return;
+    }
+
+    if (ctx->fileinfo) {
+        char js_metadata_filename[PATH_MAX];
+        if (snprintf(js_metadata_filename, sizeof(js_metadata_filename), "%s.%" PRIuMAX ".%u.json",
+                    final_filename, (uintmax_t)SCTIME_SECS(p->ts),
+                    ff->file_store_id) == (int)sizeof(js_metadata_filename)) {
+            WARN_ONCE(WOT_SNPRINTF, "Failed to write file info record. Output filename truncated.");
+        } else {
+            JsonBuilder *js_fileinfo =
+                    JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL);
+            if (likely(js_fileinfo != NULL)) {
+                jb_close(js_fileinfo);
+                FILE *out = fopen(js_metadata_filename, "w");
+                if (out != NULL) {
+                    size_t js_len = jb_len(js_fileinfo);
+                    fwrite(jb_ptr(js_fileinfo), js_len, 1, out);
+                    fclose(out);
+                }
+                jb_free(js_fileinfo);
+            }
+        }
+    }
+}
+
+static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, File *ff,
+        void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
+        uint8_t dir)
+{
+    SCEnter();
+    OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)thread_data;
+    OutputFilestoreCtx *ctx = aft->ctx;
+    char filename[PATH_MAX] = "";
+    int file_fd = -1;
+
+    SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
+
+    char base_filename[PATH_MAX] = "";
+    snprintf(base_filename, sizeof(base_filename), "%s/file.%u",
+            ctx->tmpdir, ff->file_store_id);
+    snprintf(filename, sizeof(filename), "%s", base_filename);
+
+    if (flags & OUTPUT_FILEDATA_FLAG_OPEN) {
+        file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY,
+                0644);
+        if (file_fd == -1) {
+            StatsIncr(tv, aft->fs_error_counter);
+            SCLogWarning("Filestore (v2) failed to create %s: %s", filename, strerror(errno));
+            return -1;
+        }
+
+        if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
+            SC_ATOMIC_ADD(filestore_open_file_cnt, 1);
+            ff->fd = file_fd;
+        } else {
+            if (FileGetMaxOpenFiles() > 0) {
+                StatsIncr(tv, aft->counter_max_hits);
+            }
+            ff->fd = -1;
+        }
+    /* we can get called with a NULL ffd when we need to close */
+    } else if (data != NULL) {
+        if (ff->fd == -1) {
+            file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
+            if (file_fd == -1) {
+                StatsIncr(tv, aft->fs_error_counter);
+                WARN_ONCE(WOT_OPEN, "Filestore (v2) failed to open file %s: %s", filename,
+                        strerror(errno));
+                return -1;
+            }
+        } else {
+            file_fd = ff->fd;
+        }
+    }
+
+    if (file_fd != -1) {
+        ssize_t r = write(file_fd, (const void *)data, (size_t)data_len);
+        if (r == -1) {
+            StatsIncr(tv, aft->fs_error_counter);
+            WARN_ONCE(WOT_WRITE, "Filestore (v2) failed to write to %s: %s", filename,
+                    strerror(errno));
+            if (ff->fd != -1) {
+                SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
+            }
+            ff->fd = -1;
+        }
+        if (ff->fd == -1) {
+            close(file_fd);
+        }
+    }
+
+    if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
+        if (ff->fd != -1) {
+            close(ff->fd);
+            ff->fd = -1;
+            SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
+        }
+        OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, tx, tx_id, dir);
+    }
+
+    return 0;
+}
+
+static TmEcode OutputFilestoreLogThreadInit(ThreadVars *t, const void *initdata,
+        void **data)
+{
+    OutputFilestoreLogThread *aft = SCMalloc(sizeof(OutputFilestoreLogThread));
+    if (unlikely(aft == NULL))
+        return TM_ECODE_FAILED;
+    memset(aft, 0, sizeof(OutputFilestoreLogThread));
+
+    if (initdata == NULL) {
+        SCLogDebug("Error getting context for LogFileStore. \"initdata\" argument NULL");
+        SCFree(aft);
+        return TM_ECODE_FAILED;
+    }
+
+    OutputFilestoreCtx *ctx = ((OutputCtx *)initdata)->data;
+    aft->ctx = ctx;
+
+    aft->counter_max_hits =
+        StatsRegisterCounter("file_store.open_files_max_hit", t);
+
+    /* File system type errors (open, write, rename) will only be
+     * logged once. But this stat will be incremented for every
+     * occurrence. */
+    aft->fs_error_counter = StatsRegisterCounter("file_store.fs_errors", t);
+
+    *data = (void *)aft;
+    return TM_ECODE_OK;
+}
+
+static TmEcode OutputFilestoreLogThreadDeinit(ThreadVars *t, void *data)
+{
+    OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)data;
+    if (aft == NULL) {
+        return TM_ECODE_OK;
+    }
+
+    /* clear memory */
+    memset(aft, 0, sizeof(OutputFilestoreLogThread));
+
+    SCFree(aft);
+    return TM_ECODE_OK;
+}
+
+static void OutputFilestoreLogDeInitCtx(OutputCtx *output_ctx)
+{
+    OutputFilestoreCtx *ctx = (OutputFilestoreCtx *)output_ctx->data;
+    if (ctx->xff_cfg != NULL) {
+        SCFree(ctx->xff_cfg);
+    }
+    SCFree(ctx);
+    SCFree(output_ctx);
+}
+
+static void GetLogDirectory(const ConfNode *conf, char *out, size_t out_size)
+{
+    const char *log_base_dir = ConfNodeLookupChildValue(conf, "dir");
+    if (log_base_dir == NULL) {
+        SCLogConfig("Filestore (v2) default log directory %s", default_log_dir);
+        log_base_dir = default_log_dir;
+    }
+    if (PathIsAbsolute(log_base_dir)) {
+        strlcpy(out, log_base_dir, out_size);
+    } else {
+        const char *default_log_prefix = ConfigGetLogDirectory();
+        snprintf(out, out_size, "%s/%s", default_log_prefix, log_base_dir);
+    }
+}
+
+static bool InitFilestoreDirectory(const char *dir)
+{
+    const uint8_t dir_count = 0xff;
+
+    if (!SCPathExists(dir)) {
+        SCLogInfo("Filestore (v2) creating directory %s", dir);
+        if (SCCreateDirectoryTree(dir, true) != 0) {
+            SCLogError("Filestore (v2) failed to create directory %s: %s", dir, strerror(errno));
+            return false;
+        }
+    }
+
+    for (int i = 0; i <= dir_count; i++) {
+        char leaf[PATH_MAX];
+        int n = snprintf(leaf, sizeof(leaf), "%s/%02x", dir, i);
+        if (n < 0 || n >= PATH_MAX) {
+            SCLogError("Filestore (v2) failed to create leaf directory: "
+                       "path too long");
+            return false;
+        }
+        if (!SCPathExists(leaf)) {
+            SCLogInfo("Filestore (v2) creating directory %s", leaf);
+            if (SCDefaultMkDir(leaf) != 0) {
+                SCLogError(
+                        "Filestore (v2) failed to create directory %s: %s", leaf, strerror(errno));
+                return false;
+            }
+        }
+    }
+
+    /* Make sure the tmp directory exists. */
+    char tmpdir[PATH_MAX];
+    int n = snprintf(tmpdir, sizeof(tmpdir), "%s/tmp", dir);
+    if (n < 0 || n >= PATH_MAX) {
+        SCLogError("Filestore (v2) failed to create tmp directory: path too long");
+        return false;
+    }
+    if (!SCPathExists(tmpdir)) {
+        SCLogInfo("Filestore (v2) creating directory %s", tmpdir);
+        if (SCDefaultMkDir(tmpdir) != 0) {
+            SCLogError("Filestore (v2) failed to create directory %s: %s", tmpdir, strerror(errno));
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/** \brief Create a new http log OutputFilestoreCtx.
+ *  \param conf Pointer to ConfNode containing this loggers configuration.
+ *  \return NULL if failure, OutputFilestoreCtx* to the file_ctx if succesful
+ * */
+static OutputInitResult OutputFilestoreLogInitCtx(ConfNode *conf)
+{
+    OutputInitResult result = { NULL, false };
+
+    intmax_t version = 0;
+    if (!ConfGetChildValueInt(conf, "version", &version) || version < 2) {
+        SCLogWarning("File-store v1 has been removed. Please update to file-store v2.");
+        return result;
+    }
+
+    if (RunModeOutputFiledataEnabled()) {
+        SCLogWarning("A file data logger is already enabled. Filestore (v2) "
+                     "will not be enabled.");
+        return result;
+    }
+
+    char log_directory[PATH_MAX] = "";
+    GetLogDirectory(conf, log_directory, sizeof(log_directory));
+    if (!InitFilestoreDirectory(log_directory)) {
+        return result;
+    }
+
+    OutputFilestoreCtx *ctx = SCCalloc(1, sizeof(*ctx));
+    if (unlikely(ctx == NULL)) {
+        return result;
+    }
+
+    strlcpy(ctx->prefix, log_directory, sizeof(ctx->prefix));
+    int written = snprintf(ctx->tmpdir, sizeof(ctx->tmpdir) - 1, "%s/tmp",
+            log_directory);
+    if (written == sizeof(ctx->tmpdir)) {
+        SCLogError("File-store output directory overflow.");
+        SCFree(ctx);
+        return result;
+    }
+
+    ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, ctx->xff_cfg);
+    }
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL)) {
+        SCFree(ctx);
+        return result;
+    }
+
+    output_ctx->data = ctx;
+    output_ctx->DeInit = OutputFilestoreLogDeInitCtx;
+
+    const char *write_fileinfo = ConfNodeLookupChildValue(conf,
+            "write-fileinfo");
+    if (write_fileinfo != NULL && ConfValIsTrue(write_fileinfo)) {
+        SCLogConfig("Filestore (v2) will output fileinfo records.");
+        ctx->fileinfo = true;
+    }
+
+    const char *force_filestore = ConfNodeLookupChildValue(conf,
+            "force-filestore");
+    if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
+        FileForceFilestoreEnable();
+        SCLogInfo("forcing filestore of all files");
+    }
+
+    const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
+    if (force_magic != NULL && ConfValIsTrue(force_magic)) {
+        FileForceMagicEnable();
+        SCLogConfig("Filestore (v2) forcing magic lookup for stored files");
+    }
+
+    FileForceHashParseCfg(conf);
+
+    /* The new filestore requires SHA256. */
+    FileForceSha256Enable();
+
+    ProvidesFeature(FEATURE_OUTPUT_FILESTORE);
+
+    const char *stream_depth_str = ConfNodeLookupChildValue(conf,
+            "stream-depth");
+    if (stream_depth_str != NULL && strcmp(stream_depth_str, "no")) {
+        uint32_t stream_depth = 0;
+        if (ParseSizeStringU32(stream_depth_str,
+                               &stream_depth) < 0) {
+            SCLogError("Error parsing "
+                       "file-store.stream-depth "
+                       "from conf file - %s.  Killing engine",
+                    stream_depth_str);
+            exit(EXIT_FAILURE);
+        }
+        if (stream_depth) {
+            if (stream_depth <= stream_config.reassembly_depth) {
+                SCLogWarning("file-store.stream-depth value %" PRIu32 " has "
+                             "no effect since it's less than stream.reassembly.depth "
+                             "value.",
+                        stream_depth);
+            } else {
+                FileReassemblyDepthEnable(stream_depth);
+            }
+        }
+    }
+
+    const char *file_count_str = ConfNodeLookupChildValue(conf,
+            "max-open-files");
+    if (file_count_str != NULL) {
+        uint32_t file_count = 0;
+        if (ParseSizeStringU32(file_count_str,
+                               &file_count) < 0) {
+            SCLogError("Error parsing "
+                       "file-store.max-open-files "
+                       "from conf file - %s.  Killing engine",
+                    file_count_str);
+            exit(EXIT_FAILURE);
+        } else {
+            if (file_count != 0) {
+                FileSetMaxOpenFiles(file_count);
+                SCLogConfig("Filestore (v2) will keep a max of %d "
+                        "simultaneously open files", file_count);
+            }
+        }
+    }
+
+    result.ctx = output_ctx;
+    result.ok = true;
+    SCReturnCT(result, "OutputInitResult");
+}
+
+void OutputFilestoreRegister(void)
+{
+    OutputRegisterFiledataModule(LOGGER_FILE_STORE, MODULE_NAME, "file-store",
+            OutputFilestoreLogInitCtx, OutputFilestoreLogger,
+            OutputFilestoreLogThreadInit, OutputFilestoreLogThreadDeinit,
+            NULL);
+
+    SC_ATOMIC_INIT(filestore_open_file_cnt);
+    SC_ATOMIC_SET(filestore_open_file_cnt, 0);
+}
+
+void OutputFilestoreRegisterGlobalCounters(void)
+{
+    StatsRegisterGlobalCounter("file_store.open_files", OutputFilestoreOpenFilesCounter);
+}