Adding upstream version 1.44.3.upstream/1.44.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 110 insertions, 0 deletions

diff --git a/fluent-bit/src/multiline/flb_ml_parser_docker.c b/fluent-bit/src/multiline/flb_ml_parser_docker.c
new file mode 100644
index 00000000..5b622d32
--- /dev/null
+++ b/fluent-bit/src/multiline/flb_ml_parser_docker.c
@@ -0,0 +1,110 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2022 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/multiline/flb_ml.h>
+#include <fluent-bit/multiline/flb_ml_parser.h>
+
+/* Creates a parser for Docker */
+static struct flb_parser *docker_parser_create(struct flb_config *config)
+{
+    struct flb_parser *p;
+
+    p = flb_parser_create("_ml_json_docker",      /* parser name */
+                          "json",                 /* backend type */
+                          NULL,                   /* regex */
+                          FLB_TRUE,               /* skip_empty */
+                          "%Y-%m-%dT%H:%M:%S.%L", /* time format */
+                          "time",                 /* time key */
+                          NULL,                   /* time offset */
+                          FLB_TRUE,               /* time keep */
+                          FLB_FALSE,              /* time strict */
+                          FLB_FALSE,              /* no bare keys */
+                          NULL,                   /* parser types */
+                          0,                      /* types len */
+                          NULL,                   /* decoders */
+                          config);                /* Fluent Bit context */
+    return p;
+}
+
+/* Our first multiline mode: 'docker' */
+struct flb_ml_parser *flb_ml_parser_docker(struct flb_config *config)
+{
+    struct flb_parser *parser;
+    struct flb_ml_parser *mlp;
+
+    /* Create a Docker parser */
+    parser = docker_parser_create(config);
+    if (!parser) {
+        return NULL;
+    }
+
+    /*
+     * Let's explain this multiline mode, then you (the reader) might want
+     * to submit a PR with new built-in modes :)
+     *
+     * Containerized apps under Docker writes logs to stdout/stderr. These streams
+     * (stdout/stderr) are handled by Docker, in most of cases the content is
+     * stored in a .json file in your file system. A message like "hey!" gets into
+     * a JSON map like this:
+     *
+     * {"log": "hey!\n", "stream": "stdout", "time": "2021-02-01T01:40:03.53412Z"}
+     *
+     * By Docker log spec, any 'log' key that "ends with a \n" it's a complete
+     * log record, but Docker also limits the log record size to 16KB, so a long
+     * message that does not fit into 16KB can be split in multiple JSON lines,
+     * the following example use short words to describe the context:
+     *
+     * - original message: 'one, two, three\n'
+     *
+     * Docker log interpretation:
+     *
+     * - {"log": "one, ", "stream": "stdout", "time": "2021-02-01T01:40:03.53413Z"}
+     * - {"log": "two, ", "stream": "stdout", "time": "2021-02-01T01:40:03.53414Z"}
+     * - {"log": "three\n", "stream": "stdout", "time": "2021-02-01T01:40:03.53415Z"}
+     *
+     * So every 'log' key that does not ends with '\n', it's a partial log record
+     * and for logging purposes it needs to be concatenated with further messages
+     * until a final '\n' is found.
+     *
+     * We setup the Multiline mode as follows:
+     *
+     * - Use the type 'FLB_ML_ENDSWITH' to specify that we expect the 'log'
+     *   key must ends with a '\n' for complete messages, otherwise it means is
+     *   a continuation message. In case a message is not complete just wait until
+     *   500 milliseconds (0.5 second) and flush the buffer.
+     */
+    mlp = flb_ml_parser_create(config,                  /* Fluent Bit context */
+                               "docker",                /* name           */
+                               FLB_ML_ENDSWITH,         /* type           */
+                               "\n",                    /* match_str      */
+                               FLB_FALSE,               /* negate         */
+                               FLB_ML_FLUSH_TIMEOUT,    /* flush_ms  */
+                               "log",                   /* key_content    */
+                               "stream",                /* key_group      */
+                               NULL,                    /* key_pattern    */
+                               parser,                  /* parser ctx     */
+                               NULL);                   /* parser name    */
+    if (!mlp) {
+        flb_error("[multiline] could not create 'docker mode'");
+        return NULL;
+    }
+
+    return mlp;
+}