diff options
Diffstat (limited to 'fluent-bit/tests/internal/data/stream_processor')
19 files changed, 86 insertions, 0 deletions
diff --git a/fluent-bit/tests/internal/data/stream_processor/gen_msgpack.sh b/fluent-bit/tests/internal/data/stream_processor/gen_msgpack.sh new file mode 100755 index 00000000..19339318 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/gen_msgpack.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +echo "=== Hit ctrl-c if it runs forever!... ===" +echo + +# Delete old samples file +rm -rf samples.mp samples-subkeys.mp + +# Generate new msgpack file using as a source samples.json +../../../../build/bin/fluent-bit -R ../../../../conf/parsers.conf \ + -i tail -t samples -p path=samples.json -p parser=json \ + -i tail -t samples-subkeys -p path=samples-subkeys.json -p parser=json \ + -o file -m samples -p format=msgpack -p file=samples.mp \ + -o file -m samples-subkeys -p format=msgpack -p file=samples-subkeys.mp -f 1 & + pid=$! + sleep 2 + kill -9 $pid + + # Generate new msgpack files for hopping window + files=$(find "samples-hw/" -type f -name "*.mp" | sort) + + for file in $files + do + rm -f $file + done + + files=$(find "samples-hw/" -type f -name "*.json" | sort) + + for file in $files + do + # Generate new msgpack files for hopping window + echo "" + echo "=== Generating MessagePack file for $file ... ===" + ../../../../build/bin/fluent-bit -R ../../../../conf/parsers.conf \ + -i tail -p path=$file -p parser=json \ + -o file -p format=msgpack -p file=$(echo $file | sed s/json/mp/) -f 1 & + pid=$! + sleep 2 + kill -9 $pid + done diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.json new file mode 100644 index 00000000..45c8aea2 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.json @@ -0,0 +1,4 @@ +{"id": 0, "word1": "fluent" , "word2": "logging" , "bytes": 10 , "bool": true, "usage": 10} +{"id": 1, "word1": "fluentd" , "word2": "rlz" , "bytes": 10.0 , "bool": true, "usage": 20} +{"id": 2, "word1": "fluent-bit" , "word3": "rlz" , "bytes": 10 , "bool": true, "usage": 30} +{"id": 3, "word1": "fluent-logger" , "word3": "" , "bytes": 10 , "bool": true, "usage": 40} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.mp Binary files differnew file mode 100644 index 00000000..0ea828bd --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/1.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.json new file mode 100644 index 00000000..870a3eb4 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.json @@ -0,0 +1,4 @@ +{"id": 4, "word1": "forward" , "word3": "plain" , "bytes": 10 , "bool": true, "usage": 50} +{"id": 5, "word5": "forward-protocol", "word6": "secure" , "bytes": 10 , "bool": true, "usage": 60} +{"id": 6, "word1": "stream" , "word3": "processing" , "bytes": 10.20 , "bool": false, "usage": 70} +{"id": 7, "word1": "edge-rocks" , "word6": "" , "bytes": 10 , "bool": true, "usage": 80} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.mp Binary files differnew file mode 100644 index 00000000..a45e1c18 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/2.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.json new file mode 100644 index 00000000..f1376be7 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.json @@ -0,0 +1,3 @@ +{"id": 8, "word1": "treasure-data" , "word3": "cncf" , "bytes": 10 , "bool": true, "usage": 90} +{"id": 9, "word1": "arm" , "word3": "linux foundation", "bytes": "10.30", "bool": false, "usage": 100} +{"id": 10, "word1": "fluent-bit" , "word3": null , "bytes": 10 , "bool": true, "usage": 110} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.mp Binary files differnew file mode 100644 index 00000000..1b6cf92a --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/3.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.json new file mode 100644 index 00000000..72373944 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.json @@ -0,0 +1,4 @@ +{"id": 11, "word1": "fluent" , "word2": "logging" , "bytes": 10 , "bool": true, "usage": 120} +{"id": 12, "word1": "fluentd" , "word2": "rlz" , "bytes": 10.0 , "bool": true, "usage": 130} +{"id": 13, "word1": "fluent-bit" , "word3": "rlz" , "bytes": 10 , "bool": true, "usage": 140} +{"id": 14, "word1": "fluent-logger" , "word3": "" , "bytes": 10 , "bool": true, "usage": 150} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.mp Binary files differnew file mode 100644 index 00000000..46716103 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/4.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.json new file mode 100644 index 00000000..273820f1 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.json @@ -0,0 +1,4 @@ +{"id": 15, "word1": "forward" , "word3": "plain" , "bytes": 10 , "bool": true, "usage": 160} +{"id": 16, "word5": "forward-protocol", "word6": "secure" , "bytes": 10 , "bool": true, "usage": 170} +{"id": 17, "word1": "stream" , "word3": "processing" , "bytes": 10.20 , "bool": false, "usage": 180} +{"id": 18, "word1": "edge-rocks" , "word6": "" , "bytes": 10 , "bool": true, "usage": 190} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.mp Binary files differnew file mode 100644 index 00000000..de531bd2 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/5.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.json new file mode 100644 index 00000000..14de259b --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.json @@ -0,0 +1,3 @@ +{"id": 19, "word1": "treasure-data" , "word3": "cncf" , "bytes": 10 , "bool": true, "usage": 200} +{"id": 20, "word1": "arm" , "word3": "linux foundation", "bytes": "10.30", "bool": false, "usage": 210} +{"id": 21, "word1": "fluent-bit" , "word3": null , "bytes": 10 , "bool": true, "usage": 220} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.mp Binary files differnew file mode 100644 index 00000000..cfe85fc7 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/6.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.json b/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.json new file mode 100644 index 00000000..c30653ee --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.json @@ -0,0 +1,4 @@ +{"id": 22, "word1": "fluent" , "word2": "logging" , "bytes": 10 , "bool": true, "usage": 230} +{"id": 23, "word1": "fluentd" , "word2": "rlz" , "bytes": 10.0 , "bool": true, "usage": 240} +{"id": 24, "word1": "fluent-bit" , "word3": "rlz" , "bytes": 10 , "bool": true, "usage": 250} +{"id": 25, "word1": "fluent-logger" , "word3": "" , "bytes": 10 , "bool": true, "usage": 260} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.mp b/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.mp Binary files differnew file mode 100644 index 00000000..19f5ccf5 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-hw/7.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.json b/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.json new file mode 100644 index 00000000..ef4c2ba5 --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.json @@ -0,0 +1,9 @@ +{"id": 0, "map": {"sub1": {"sub2": {"color": "blue"}}}} +{"id": 1, "map": {"sub1": {"sub2": {"color": "red"}, "sub4": "circle"}}} +{"id": 2, "map": {"sub1": {"sub2": 123}}} +{"id": 3, "map": {"sub1": {"sub2": "123", "stype": "a"}, "mtype": 0}} +{"id": 4, "map": {"color": "blue"}} +{"id": 5, "map": {"sub1": {"sub3": "100", "stype": "a", "sub4": "circle"}, "mtype": 0}} +{"id": 6, "map": {"sub1": {"sub3": "0.50", "stype": "b", "sub4": "rectangle"}, "mtype": 1}} +{"id": 7, "map": {"sub1": {"sub3": "5.50", "stype": "a", "sub4": "triangle"}, "mtype": 0}} +{"id": 8, "map": {"sub1": {"sub3": "10.50", "stype": "b", "sub4": "rectangle"}, "mtype": 2}} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.mp b/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.mp Binary files differnew file mode 100644 index 00000000..374e16cd --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples-subkeys.mp diff --git a/fluent-bit/tests/internal/data/stream_processor/samples.json b/fluent-bit/tests/internal/data/stream_processor/samples.json new file mode 100644 index 00000000..2cfdef6e --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples.json @@ -0,0 +1,11 @@ +{"id": 0, "word1": "fluent" , "word2": "logging" , "bytes": 10 , "bool": true, "usage": 10} +{"id": 1, "word1": "fluentd" , "word2": "rlz" , "bytes": 10.0 , "bool": true, "usage": 20} +{"id": 2, "word1": "fluent-bit" , "word3": "rlz" , "bytes": 10 , "bool": true, "usage": 30} +{"id": 3, "word1": "fluent-logger" , "word3": "" , "bytes": 10 , "bool": true, "usage": 40} +{"id": 4, "word1": "forward" , "word3": "plain" , "bytes": 10 , "bool": true, "usage": 50} +{"id": 5, "word5": "forward-protocol", "word6": "secure" , "bytes": 10 , "bool": true, "usage": 60} +{"id": 6, "word1": "stream" , "word3": "processing" , "bytes": 10.20 , "bool": false, "usage": 70} +{"id": 7, "word1": "edge-rocks" , "word6": "" , "bytes": 10 , "bool": true, "usage": 80} +{"id": 8, "word1": "treasure-data" , "word3": "cncf" , "bytes": 10 , "bool": true, "usage": 90} +{"id": 9, "word1": "arm" , "word3": "linux foundation", "bytes": "10.30", "bool": false, "usage": 100} +{"id": 10, "word1": "fluent-bit" , "word3": null , "bytes": 10 , "bool": true, "usage": 110} diff --git a/fluent-bit/tests/internal/data/stream_processor/samples.mp b/fluent-bit/tests/internal/data/stream_processor/samples.mp Binary files differnew file mode 100644 index 00000000..8304065d --- /dev/null +++ b/fluent-bit/tests/internal/data/stream_processor/samples.mp |