Merging upstream version 1.45.3+dfsg.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 126 insertions, 0 deletions

diff --git a/src/fluent-bit/scripts/append_tag.lua b/src/fluent-bit/scripts/append_tag.lua
new file mode 100644
index 000000000..c19d97a21
--- /dev/null
+++ b/src/fluent-bit/scripts/append_tag.lua
@@ -0,0 +1,5 @@
+function append_tag(tag, timestamp, record)
+    new_record = record
+    new_record["tag"] = tag
+    return 1, timestamp, new_record
+end
diff --git a/src/fluent-bit/scripts/override_time.lua b/src/fluent-bit/scripts/override_time.lua
new file mode 100644
index 000000000..a8f42b15f
--- /dev/null
+++ b/src/fluent-bit/scripts/override_time.lua
@@ -0,0 +1,23 @@
+--[[
+   This Lua script is to override timestamp with integer/float epoch time.
+   https://github.com/fluent/fluent-bit/issues/662
+
+   sample input is
+   [XXXXX.XXXXX, {"KEY_OF_TIMESTAMP"=>1530239065.807368, "data"=>"sample"}]
+   
+   expected output is
+   [1530239065.807368040, {"KEY_OF_TIMESTAMP"=>1530239065.807368, "data"=>"sample"}]
+
+
+   sample configuration:
+   [FILTER]
+    Name lua
+    Match *.*
+    script override_time.lua
+    call   override_time
+]]
+
+function override_time(tag, timestamp, record)
+         -- modify KEY_OF_TIMESTAMP properly.
+         return 1, record["KEY_OF_TIMESTAMP"], record
+end
\ No newline at end of file
diff --git a/src/fluent-bit/scripts/rate_limit.lua b/src/fluent-bit/scripts/rate_limit.lua
new file mode 100644
index 000000000..1474d113d
--- /dev/null
+++ b/src/fluent-bit/scripts/rate_limit.lua
@@ -0,0 +1,42 @@
+--[[
+   This Lua script is to do the rate limiting of logs based on some key. The Throttle filter in fluent-bit doesn't allow to do the rate limiting based on key
+
+   sample configuration:
+    [FILTER]
+     Name lua
+     Match kube.*
+     script rate_limit.lua
+     call rate_limit
+]]
+
+local counter = {}
+local time = 0
+local group_key = "docker_id" -- Used to group logs. Groups are rate limited independently.
+local group_bucket_period_s = 60 -- This is the period of of time in seconds over which group_bucket_limit applies.
+local group_bucket_limit = 6000 -- Maximum number logs allowed per groups over the period of group_bucket_period_s.
+
+-- with above values, each and every containers running on the kubernetes will have a limit of 6000 logs for every 60 seconds since contianers have unique kubernetes.docker_id value
+
+local function get_current_time(timestamp)
+    return math.floor(timestamp / group_bucket_period_s)
+end
+
+function rate_limit(tag, timestamp, record)
+    local t = os.time()
+    local current_time = get_current_time(t)
+    if current_time ~= time then
+        time = current_time
+        counter = {} -- reset the counter
+    end
+    local counter_key = record["kubernetes"][group_key]
+    local logs_count = counter[counter_key]
+    if logs_count == nil then
+        counter[counter_key] = 1
+    else
+        counter[counter_key] = logs_count + 1
+        if counter[counter_key] > group_bucket_limit then -- check if the number of logs is greater than group_bucket_limit
+            return -1, 0, 0 -- drop the log
+        end
+    end
+    return 0, 0, 0 -- keep the log
+end
\ No newline at end of file
diff --git a/src/fluent-bit/scripts/test.lua b/src/fluent-bit/scripts/test.lua
new file mode 100644
index 000000000..91ff2f1a9
--- /dev/null
+++ b/src/fluent-bit/scripts/test.lua
@@ -0,0 +1,56 @@
+--[[
+
+   This Lua script provides 3 interfaces or callbacks for filter_lua:
+
+   - cb_print   => Print records to the standard output
+   - cb_drop    => Drop the record
+   - cb_replace => Replace record content with a new table
+
+   The key inside each function is to do a proper handling of the
+   return values. Each function must return 3 values:
+
+      return code, timestamp, record
+
+   where:
+
+   - code     : -1 record must be deleted
+                 0 record not modified, keep the original
+                 1 record was modified, replace timestamp and record.
+                 2 record was modified, replace record and keep timestamp.
+   - timestamp: Unix timestamp with precision (double)
+   - record   : Table with multiple key/val
+
+   Uppon return if code == 1 (modified), then filter_lua plugin
+   will replace the original timestamp and record with the returned
+   values. If code == 0 the original record is kept otherwise if
+   code == -1, the original record will be deleted.
+]]
+
+-- Print record to the standard output
+function cb_print(tag, timestamp, record)
+   output = tag .. ":  [" .. string.format("%f", timestamp) .. ", { "
+
+   for key, val in pairs(record) do
+      output = output .. string.format(" %s => %s,", key, val)
+   end
+   
+   output = string.sub(output,1,-2) .. " }]"
+   print(output)
+
+   -- Record not modified so 'code' return value is 0 (first parameter)
+   return 0, 0, 0
+end
+
+-- Drop the record
+function cb_drop(tag, timestamp, record)
+   return -1, 0, 0
+end
+
+-- Compose a new JSON map and report it
+function cb_replace(tag, timestamp, record)
+   -- Record modified, so 'code' return value (first parameter) is 1
+   new_record = {}
+   new_record["new"] = 12345
+   new_record["old"] = record
+   return 1, timestamp, new_record
+end