diff options
Diffstat (limited to 'fluent-bit/examples')
54 files changed, 1911 insertions, 0 deletions
diff --git a/fluent-bit/examples/CMakeLists.txt b/fluent-bit/examples/CMakeLists.txt new file mode 100644 index 00000000..92f98d4f --- /dev/null +++ b/fluent-bit/examples/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(hello_world) +if(FLB_OUT_LIB) + add_subdirectory(out_lib) +endif() + diff --git a/fluent-bit/examples/filter_rust/Cargo.lock b/fluent-bit/examples/filter_rust/Cargo.lock new file mode 100644 index 00000000..26788709 --- /dev/null +++ b/fluent-bit/examples/filter_rust/Cargo.lock @@ -0,0 +1,219 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + +[[package]] +name = "filter_rust" +version = "0.1.0" +dependencies = [ + "chrono", + "libc", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + +[[package]] +name = "proc-macro2" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "serde" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "unicode-ident" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/fluent-bit/examples/filter_rust/Cargo.toml b/fluent-bit/examples/filter_rust/Cargo.toml new file mode 100644 index 00000000..924c9bdf --- /dev/null +++ b/fluent-bit/examples/filter_rust/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "filter_rust" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +serde = { version = "*", features = ["derive"] } +serde_json = "1.0" +serde_bytes = "0.11" +rmp-serde = "1.1" +chrono = "0.4" +libc = "0.2" diff --git a/fluent-bit/examples/filter_rust/README.md b/fluent-bit/examples/filter_rust/README.md new file mode 100644 index 00000000..f1f3841a --- /dev/null +++ b/fluent-bit/examples/filter_rust/README.md @@ -0,0 +1,59 @@ +# Fluent Bit / filter_rust + +This source source tree provides an example of WASM program with WASI mode mainly written in Rust. + +## Prerequisites + +* Rust + * rustc 1.61.0 (fe5b13d68 2022-05-18) +* [rustup](https://rustup.rs/) (For preparing rust compiler and toolchains) + +## How to build + +Add `wasm32-unknown-unknown` target for Rust toolchain: + +```console +$ rustup target add wasm32-unknown-unknown +``` + +Then, execute _cargo build_ as follows: + +```console +$ cargo build --target wasm32-unknown-unknown --release +``` + +Finally, `*.wasm` file will be created: + +```console +$ ls target/wasm32-unknown-unknown/release/*.wasm +target/wasm32-unknown-unknown/release/filter_rust.wasm +``` + +## How to confirm WASM filter integration + +Create fluent-bit configuration file as follows: + +```ini +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + HTTP_Server Off + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name dummy + Tag dummy.local + +[FILTER] + Name wasm + match dummy.* + WASM_Path /path/to/filter_rust.wasm + Function_Name rust_filter + accessible_paths .,/path/to/fluent-bit + +[OUTPUT] + Name stdout + Match * +``` diff --git a/fluent-bit/examples/filter_rust/src/lib.rs b/fluent-bit/examples/filter_rust/src/lib.rs new file mode 100644 index 00000000..c409ba53 --- /dev/null +++ b/fluent-bit/examples/filter_rust/src/lib.rs @@ -0,0 +1,32 @@ +// Import pure and fast JSON library written in Rust +use serde_json::json; +use serde_json::Value; +// Import chrono library to handle time related operation conveniently +use chrono::{Utc, TimeZone}; +use std::slice; +use std::str; +use std::os::raw::c_char; +use std::io::Write; + +#[no_mangle] +pub extern "C" fn rust_filter(tag: *const c_char, tag_len: u32, time_sec: u32, time_nsec: u32, record: *const c_char, record_len: u32) -> *const u8 { + let slice_tag: &[u8] = unsafe { slice::from_raw_parts(tag as *const u8, tag_len as usize) }; + let slice_record: &[u8] = unsafe { slice::from_raw_parts(record as *const u8, record_len as usize) }; + let mut vt: Vec<u8> = Vec::new(); + vt.write(slice_tag).expect("Unable to write"); + let vtag = str::from_utf8(&vt).unwrap(); + let v: Value = serde_json::from_slice(slice_record).unwrap(); + let dt = Utc.timestamp(time_sec as i64, time_nsec); + let time = dt.format("%Y-%m-%dT%H:%M:%S.%9f %z").to_string(); + + let message = json!({ + "message": v["message"], + "time": format!("{}", time), + "tag": vtag, + "original": v.to_string(), + "lang": "Rust", + }); + + let buf: String = message.to_string(); + buf.as_ptr() +} diff --git a/fluent-bit/examples/filter_rust_clib/Cargo.lock b/fluent-bit/examples/filter_rust_clib/Cargo.lock new file mode 100644 index 00000000..577860a8 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/Cargo.lock @@ -0,0 +1,219 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + +[[package]] +name = "filter_rust_clib" +version = "0.1.0" +dependencies = [ + "chrono", + "libc", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + +[[package]] +name = "proc-macro2" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "serde" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "unicode-ident" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/fluent-bit/examples/filter_rust_clib/Cargo.toml b/fluent-bit/examples/filter_rust_clib/Cargo.toml new file mode 100644 index 00000000..400508d2 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "filter_rust_clib" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "staticlib", "rlib"] + +[dependencies] +serde = { version = "*", features = ["derive"] } +serde_json = "1.0" +serde_bytes = "0.11" +rmp-serde = "1.1" +chrono = "0.4" +libc = "0.2" diff --git a/fluent-bit/examples/filter_rust_clib/Makefile b/fluent-bit/examples/filter_rust_clib/Makefile new file mode 100644 index 00000000..c8d7f429 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/Makefile @@ -0,0 +1,25 @@ +WASI_SDK_ROOT ?= /opt/wasi-sdk +STACK_SIZE ?= 8192 +INITIAL_MEMORY_SIZE ?= 65536 + +rustclib: + cargo build --target wasm32-unknown-unknown --release + +build: wasm rustclib + +wasm: rustclib rust_clib_filter.h + ${WASI_SDK_ROOT}/bin/clang -O3 -nostdlib \ + -z stack-size=${STACK_SIZE} -Wl,--initial-memory=${INITIAL_MEMORY_SIZE} \ + -o rust_clib_filter.wasm rust_clib_filter.c \ + -L ./target/wasm32-unknown-unknown/release -lfilter_rust_clib \ + -Wl,--export=__heap_base -Wl,--export=__data_end -Wl,--export=rust_clib_filter \ + -Wl,--no-entry -Wl,--strip-all -Wl,--allow-undefined + +rust_clib_filter.h: + cbindgen --crate filter_rust_clib --output filter_rust_clib.h --lang c + + +clean: + rm -f *.wasm + rm -f *.h + cargo clean diff --git a/fluent-bit/examples/filter_rust_clib/README.md b/fluent-bit/examples/filter_rust_clib/README.md new file mode 100644 index 00000000..9be89598 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/README.md @@ -0,0 +1,77 @@ +# Fluent Bit / filter_rust_clib + +This source source tree provides an example of WASM program with WASI mode mainly written in Rust. + +## Prerequisites + +* Rust + * rustc 1.61.0 (fe5b13d68 2022-05-18) +* [rustup](https://rustup.rs/) (For preparing rust compiler and toolchains) +* [cbindgen](https://github.com/eqrion/cbindgen) (For preparing C headers to include exported Rust's c style function(s)) +* [WASI SDK](https://github.com/WebAssembly/wasi-sdk) + +For Ubuntu, it's easy to install with: + +```console +$ export WASI_VERSION=14 +$ export WASI_VERSION_FULL=${WASI_VERSION}.0 +$ wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +$ sudo mkdir -p /opt/wasi-sdk/ +$ sudo tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz --strip-components=1 -C /opt/wasi-sdk +``` + +## How to build + +Add `wasm32-unknown-unknown` target for Rust toolchain: + +```console +$ rustup target add wasm32-unknown-unknown +``` + +Install `cbindgen` command as follows: + +```console +$ cargo install --force cbindgen +``` + +Then, execute _make build_ as follows: + +```console +$ make build +``` + +Finally, `*.wasm` file will be created: + +```console +$ ls *.wasm +rust_clib_filter.wasm +``` + +## How to confirm WASI integration + +Create fluent-bit configuration file as follows: + +```ini +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + HTTP_Server Off + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name dummy + Tag dummy.local + +[FILTER] + Name wasm + match dummy.* + WASM_Path /path/to/rust_clib_filter.wasm + Function_Name rust_clib_filter + accessible_paths .,/path/to/fluent-bit + +[OUTPUT] + Name stdout + Match * +``` diff --git a/fluent-bit/examples/filter_rust_clib/rust_clib_filter.c b/fluent-bit/examples/filter_rust_clib/rust_clib_filter.c new file mode 100644 index 00000000..ca19d516 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/rust_clib_filter.c @@ -0,0 +1,9 @@ +#include <stdio.h> +#include <string.h> +#include <time.h> +#include "filter_rust_clib.h" + +char* rust_clib_filter(char* tag, int len, uint32_t sec, uint32_t nsec, char* record, int record_len) +{ + return (char *)rust_filter(tag, strlen(tag), sec, nsec, record, strlen(record)); +} diff --git a/fluent-bit/examples/filter_rust_clib/src/lib.rs b/fluent-bit/examples/filter_rust_clib/src/lib.rs new file mode 100644 index 00000000..c409ba53 --- /dev/null +++ b/fluent-bit/examples/filter_rust_clib/src/lib.rs @@ -0,0 +1,32 @@ +// Import pure and fast JSON library written in Rust +use serde_json::json; +use serde_json::Value; +// Import chrono library to handle time related operation conveniently +use chrono::{Utc, TimeZone}; +use std::slice; +use std::str; +use std::os::raw::c_char; +use std::io::Write; + +#[no_mangle] +pub extern "C" fn rust_filter(tag: *const c_char, tag_len: u32, time_sec: u32, time_nsec: u32, record: *const c_char, record_len: u32) -> *const u8 { + let slice_tag: &[u8] = unsafe { slice::from_raw_parts(tag as *const u8, tag_len as usize) }; + let slice_record: &[u8] = unsafe { slice::from_raw_parts(record as *const u8, record_len as usize) }; + let mut vt: Vec<u8> = Vec::new(); + vt.write(slice_tag).expect("Unable to write"); + let vtag = str::from_utf8(&vt).unwrap(); + let v: Value = serde_json::from_slice(slice_record).unwrap(); + let dt = Utc.timestamp(time_sec as i64, time_nsec); + let time = dt.format("%Y-%m-%dT%H:%M:%S.%9f %z").to_string(); + + let message = json!({ + "message": v["message"], + "time": format!("{}", time), + "tag": vtag, + "original": v.to_string(), + "lang": "Rust", + }); + + let buf: String = message.to_string(); + buf.as_ptr() +} diff --git a/fluent-bit/examples/filter_wasm_c/Makefile b/fluent-bit/examples/filter_wasm_c/Makefile new file mode 100644 index 00000000..77d8772a --- /dev/null +++ b/fluent-bit/examples/filter_wasm_c/Makefile @@ -0,0 +1,15 @@ +WASI_SDK_ROOT ?= /opt/wasi-sdk +STACK_SIZE ?= 8192 +INITIAL_MEMORY_SIZE ?= 65536 + +.PHONY: wasm + +wasm: + ${WASI_SDK_ROOT}/bin/clang -O3 -nostdlib \ + -z stack-size=${STACK_SIZE} -Wl,--initial-memory=${INITIAL_MEMORY_SIZE} \ + -o c_filter.wasm c_filter.c \ + -Wl,--export=__heap_base -Wl,--export=__data_end -Wl,--export=c_filter \ + -Wl,--no-entry -Wl,--strip-all -Wl,--allow-undefined + +clean: + rm -f *.wasm diff --git a/fluent-bit/examples/filter_wasm_c/README.md b/fluent-bit/examples/filter_wasm_c/README.md new file mode 100644 index 00000000..64c06e5f --- /dev/null +++ b/fluent-bit/examples/filter_wasm_c/README.md @@ -0,0 +1,69 @@ +# Fluent Bit / filter_wasm_c + +This source source tree provides an example of WASM filter program with WASI mode. + +## Prerequisites + +Tested on + +* [WASI SDK](https://github.com/WebAssembly/wasi-sdk) + +For Ubuntu, it's easy to install with: + +```console +$ export WASI_VERSION=14 +$ export WASI_VERSION_FULL=${WASI_VERSION}.0 +$ wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +$ sudo mkdir -p /opt/wasi-sdk/ +$ sudo tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz --strip-components=1 -C /opt/wasi-sdk +``` + +## How to build + +Execute _make_ as follows: + +```console +$ make +/opt/wasi-sdk/bin/clang -O3 -nostdlib \ + -z stack-size=8192 -Wl,--initial-memory=65536 \ + -o c_filter.wasm c_filter.c \ + -Wl,--export=__heap_base -Wl,--export=__data_end -Wl,--export=c_filter \ + -Wl,--no-entry -Wl,--strip-all -Wl,--allow-undefined +``` + +Finally, under the same directory, `*.wasm` file will be created: + +```console +$ ls *.wasm +c_filter.wasm +``` + +## How to confirm WASM filter integration + +Create fluent-bit configuration file as follows: + +```ini +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + HTTP_Server Off + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name dummy + Tag dummy.local + +[FILTER] + Name wasm + match dummy.* + WASM_Path c_filter.wasm + Function_Name c_filter + accessible_paths .,/path/to/fluent-bit + +[OUTPUT] + Name stdout + Match * + +``` diff --git a/fluent-bit/examples/filter_wasm_c/c_filter.c b/fluent-bit/examples/filter_wasm_c/c_filter.c new file mode 100644 index 00000000..53a45c20 --- /dev/null +++ b/fluent-bit/examples/filter_wasm_c/c_filter.c @@ -0,0 +1,22 @@ +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <time.h> + +char* c_filter(char* tag, int len, uint32_t sec, uint32_t nsec, char* record, int record_len) +{ + char *buf; + buf = malloc(1024); + if (!buf) { + printf("malloc buf failed\n"); + return NULL; + } + + struct timespec ts; + ts.tv_sec = sec; + ts.tv_nsec = nsec; + + sprintf(buf, "{\"tag\":\"%s\",\"time\":\"%lld.%9ld\", \"message\":\"Hello, from C!\", \"original\": %s}", + tag, (long long)ts.tv_sec, ts.tv_nsec, record); + return buf; +} diff --git a/fluent-bit/examples/filter_wasm_go/README.md b/fluent-bit/examples/filter_wasm_go/README.md new file mode 100644 index 00000000..863cf836 --- /dev/null +++ b/fluent-bit/examples/filter_wasm_go/README.md @@ -0,0 +1,63 @@ +# Fluent Bit / filter_wasm_go + +This source source tree provides an example of WASM filter program with WASI mode. + +## Prerequisites + +Tested on + +* TinyGo + * [tinygo](https://tinygo.org/) tinygo version 0.23.0 linux/amd64 (using go version go1.18.2 and LLVM version 14.0.0) + * [tinygo](https://tinygo.org/) tinygo version 0.24.0 linux/amd64 (using go version go1.18.2 and LLVM version 14.0.0) + +For Ubuntu, it's easy to install with: + +```console +$ wget https://github.com/tinygo-org/tinygo/releases/download/v0.24.0/tinygo_0.24.0_amd64.deb +$ sudo dpkg -i tinygo_0.24.0_amd64.deb +``` + +## How to build + +Execute _tinygo build_ as follows: + +```console +$ tinygo build -wasm-abi=generic -target=wasi -o filter.wasm filter.go +``` + +Finally, under the same directory, `*.wasm` file will be created: + +```console +$ ls *.wasm +filter.wasm +``` + +## How to confirm WASM filter integration + +Create fluent-bit configuration file as follows: + +```ini +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + HTTP_Server Off + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name dummy + Tag dummy.local + +[FILTER] + Name wasm + match dummy.* + WASM_Path /path/to/filter.wasm + Function_Name go_filter + accessible_paths .,/path/to/fluent-bit + +[OUTPUT] + Name stdout + Match * + +``` diff --git a/fluent-bit/examples/filter_wasm_go/filter.go b/fluent-bit/examples/filter_wasm_go/filter.go new file mode 100644 index 00000000..fe7303c6 --- /dev/null +++ b/fluent-bit/examples/filter_wasm_go/filter.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "time" + "unsafe" + + "github.com/valyala/fastjson" +) + +//export go_filter +func go_filter(tag *uint8, tag_len uint, time_sec uint, time_nsec uint, record *uint8, record_len uint) *uint8 { + btag := unsafe.Slice(tag, tag_len) // Note, requires Go 1.17 (tinygo 0.20) + brecord := unsafe.Slice(record, record_len) + now := time.Unix(int64(time_sec), int64(time_nsec)) + + br := string(brecord) + var p fastjson.Parser + value, err := p.Parse(br) + if err != nil { + fmt.Println(err) + return nil + } + obj, err := value.Object() + if err != nil { + fmt.Println(err) + return nil + } + + var arena fastjson.Arena + obj.Set("time", arena.NewString(now.String())) + obj.Set("tag", arena.NewString(string(btag))) + obj.Set("original", arena.NewString(br)) + s := obj.String() + s += string(rune(0)) // Note: explicit null terminator. + rv := []byte(s) + + return &rv[0] +} + +func main() {} diff --git a/fluent-bit/examples/filter_wasm_go/go.mod b/fluent-bit/examples/filter_wasm_go/go.mod new file mode 100644 index 00000000..8250a72d --- /dev/null +++ b/fluent-bit/examples/filter_wasm_go/go.mod @@ -0,0 +1,9 @@ +module filter_wasm_go + +go 1.18 + +require ( + github.com/valyala/fastjson v1.6.3 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect +) diff --git a/fluent-bit/examples/filter_wasm_go/go.sum b/fluent-bit/examples/filter_wasm_go/go.sum new file mode 100644 index 00000000..2489b8d5 --- /dev/null +++ b/fluent-bit/examples/filter_wasm_go/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc= +github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= +github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/fluent-bit/examples/hello_world/CMakeLists.txt b/fluent-bit/examples/hello_world/CMakeLists.txt new file mode 100644 index 00000000..2e55b75f --- /dev/null +++ b/fluent-bit/examples/hello_world/CMakeLists.txt @@ -0,0 +1,9 @@ +set(src + ${src} + hello_world.c + ) + +find_package (Threads) +add_executable(hello_world ${src}) +target_link_libraries(hello_world fluent-bit-shared) +target_link_libraries(hello_world ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fluent-bit/examples/hello_world/README.md b/fluent-bit/examples/hello_world/README.md new file mode 100644 index 00000000..f9fc5a4e --- /dev/null +++ b/fluent-bit/examples/hello_world/README.md @@ -0,0 +1,6 @@ +# Fluent Bit / Hello World + +This tool uses the Fluent Bit library version to flush data to one of the +registered outputs. + +When using Fluent Bit as a library means the caller becomes the input. diff --git a/fluent-bit/examples/hello_world/hello_world.c b/fluent-bit/examples/hello_world/hello_world.c new file mode 100644 index 00000000..f4925bf6 --- /dev/null +++ b/fluent-bit/examples/hello_world/hello_world.c @@ -0,0 +1,60 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit Demo + * =============== + * Copyright (C) 2015 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> + +int main() +{ + int i; + int n; + char tmp[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Initialize library */ + ctx = flb_create(); + if (!ctx) { + exit(EXIT_FAILURE); + } + + in_ffd = flb_input(ctx, "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, "stdout", NULL); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + /* Start the background worker */ + flb_start(ctx); + + /* Push some data */ + for (i = 0; i < 100; i++) { + n = snprintf(tmp, sizeof(tmp) - 1, + "[%f, {\"key\": \"val %i\"}]", + flb_time_now(), i); + flb_lib_push(ctx, in_ffd, tmp, n); + } + + flb_stop(ctx); + + /* Release Resources */ + flb_destroy(ctx); + + return 0; +} diff --git a/fluent-bit/examples/kafka_filter/.env b/fluent-bit/examples/kafka_filter/.env new file mode 100644 index 00000000..61c27af8 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/.env @@ -0,0 +1,4 @@ +KAFKA_HOST=kafka-broker +KAFKA_PORT=9092 +ZOOKEEPER_HOST=zookeeper +ZOOKEEPER_PORT=2181 diff --git a/fluent-bit/examples/kafka_filter/.gitignore b/fluent-bit/examples/kafka_filter/.gitignore new file mode 100644 index 00000000..f8edd2f2 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/.gitignore @@ -0,0 +1,2 @@ +build/ +kafka/ diff --git a/fluent-bit/examples/kafka_filter/Dockerfile b/fluent-bit/examples/kafka_filter/Dockerfile new file mode 100644 index 00000000..5a18e63e --- /dev/null +++ b/fluent-bit/examples/kafka_filter/Dockerfile @@ -0,0 +1,44 @@ +FROM debian:bullseye-slim as builder +ENV DEBIAN_FRONTEND noninteractive +ENV KAFKA_URL https://downloads.apache.org/kafka/3.4.1/kafka_2.13-3.4.1.tgz +ENV KAFKA_SHA256 a76f17a52b8f2cd31de11571ff366a714820b6e4e02893b0159d09db0edaf998 + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + cmake \ + pkg-config \ + libsasl2-dev \ + libssl-dev \ + flex \ + openjdk-11-jre-headless \ + bison \ + netcat-openbsd \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download kafka to access "kafka-topics" script +WORKDIR /kafka +RUN bash -c 'curl -L $KAFKA_URL | tee kafka.tgz | sha256sum -c <(echo "$KAFKA_SHA256 -")' \ + && tar --strip-components=1 -xf kafka.tgz \ + && mv /kafka/bin/kafka-topics.sh /kafka/bin/kafka-topics +ENV PATH="${PATH}:/kafka/bin" + +WORKDIR /build/ +COPY . /source +RUN cmake -DFLB_DEV=On \ + -DFLB_IN_KAFKA=On \ + -DFLB_OUT_KAFKA=On \ + -DFLB_CONFIG_YAML=Off \ + /source && \ + cmake --build . --parallel + +FROM builder as runner +COPY --from=builder /build/bin/fluent-bit /usr/local/bin/fluent-bit +COPY examples/kafka_filter/kafka.conf /etc/kafka.conf +COPY examples/kafka_filter/kafka.lua /etc/kafka.lua +CMD ["/usr/local/bin/fluent-bit", "-c", "/etc/kafka.conf"] diff --git a/fluent-bit/examples/kafka_filter/Makefile b/fluent-bit/examples/kafka_filter/Makefile new file mode 100644 index 00000000..75b4b3a8 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/Makefile @@ -0,0 +1,11 @@ +build: + docker compose build + +start: + docker compose up -d + docker compose logs -f fluent-bit kafka-consumer + +stop: + docker compose down + +.PHONY: build start stop diff --git a/fluent-bit/examples/kafka_filter/docker-compose.yml b/fluent-bit/examples/kafka_filter/docker-compose.yml new file mode 100644 index 00000000..dc758c9e --- /dev/null +++ b/fluent-bit/examples/kafka_filter/docker-compose.yml @@ -0,0 +1,68 @@ +version: '3.4' + +x-service-common-fields: &service-common-fields + volumes: + - ./scripts:/scripts + env_file: + - ./.env + + +services: + fluent-bit: + <<: *service-common-fields + build: + context: ../.. + target: runner + dockerfile: examples/kafka_filter/Dockerfile + command: /scripts/flb-start.sh + depends_on: + - kafka-consumer + + + kafka-create-topics: + <<: *service-common-fields + image: confluentinc/cp-server:7.0.1 + command: /scripts/create-topics.sh + depends_on: + - kafka-broker + + + kafka-producer: + <<: *service-common-fields + image: confluentinc/cp-server:7.0.1 + command: /scripts/kafka-produce.sh + depends_on: + - fluent-bit + + + kafka-consumer: + <<: *service-common-fields + image: confluentinc/cp-server:7.0.1 + command: /scripts/kafka-consume.sh + depends_on: + - kafka-create-topics + + + kafka-broker: + image: confluentinc/cp-server:7.0.1 + hostname: broker + container_name: kafka-broker + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: "${ZOOKEEPER_HOST}:${ZOOKEEPER_PORT}" + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_HOST}:${KAFKA_PORT}" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + + + zookeeper: + image: confluentinc/cp-zookeeper:7.0.1 + hostname: zookeeper + container_name: zookeeper + environment: + ZOOKEEPER_CLIENT_PORT: "${ZOOKEEPER_PORT}" + ZOOKEEPER_TICK_TIME: 2000 diff --git a/fluent-bit/examples/kafka_filter/kafka.conf b/fluent-bit/examples/kafka_filter/kafka.conf new file mode 100644 index 00000000..2c3daf78 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/kafka.conf @@ -0,0 +1,22 @@ +[SERVICE] + Flush 0.1 + Grace 2 + Log_Level info + +[INPUT] + Name kafka + brokers kafka-broker:9092 + topics fb-source + poll_ms 100 + format json + +[FILTER] + Name lua + Match * + script kafka.lua + call modify_kafka_message + +[OUTPUT] + Name kafka + brokers kafka-broker:9092 + topics fb-sink diff --git a/fluent-bit/examples/kafka_filter/kafka.lua b/fluent-bit/examples/kafka_filter/kafka.lua new file mode 100644 index 00000000..3c6d9dc1 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/kafka.lua @@ -0,0 +1,8 @@ +local count = 0 +function modify_kafka_message(tag, timestamp, record) + count = count + 1 + local payload = record.payload + payload.topic = record.topic + payload.status = 'processed by fluent-bit, total records: '..tostring(count) + return 1, timestamp, payload +end diff --git a/fluent-bit/examples/kafka_filter/scripts/common.sh b/fluent-bit/examples/kafka_filter/scripts/common.sh new file mode 100644 index 00000000..d9a2bc6f --- /dev/null +++ b/fluent-bit/examples/kafka_filter/scripts/common.sh @@ -0,0 +1,17 @@ +#!/bin/bash -ue + +wait_kafka() { + while ! nc -z "$KAFKA_HOST" "$KAFKA_PORT"; do + sleep 0.1 + done +} + +wait_topic() { + wait_kafka + local topic=$1 + [ -z "$topic" ] && return 1 + while true; do + kafka-topics --list --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT" | grep -q "^$topic$" && break + sleep 0.1 + done +} diff --git a/fluent-bit/examples/kafka_filter/scripts/create-topics.sh b/fluent-bit/examples/kafka_filter/scripts/create-topics.sh new file mode 100755 index 00000000..a7adbddf --- /dev/null +++ b/fluent-bit/examples/kafka_filter/scripts/create-topics.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ue + +# shellcheck disable=SC1091 +. /scripts/common.sh + +wait_kafka + +kafka-topics --create --partitions 1 --replication-factor 1 --topic fb-source --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT" +kafka-topics --create --partitions 1 --replication-factor 1 --topic fb-sink --bootstrap-server "$KAFKA_HOST:$KAFKA_PORT" diff --git a/fluent-bit/examples/kafka_filter/scripts/flb-start.sh b/fluent-bit/examples/kafka_filter/scripts/flb-start.sh new file mode 100755 index 00000000..56e5ee0d --- /dev/null +++ b/fluent-bit/examples/kafka_filter/scripts/flb-start.sh @@ -0,0 +1,8 @@ +#!/bin/bash -ue + +# shellcheck disable=SC1091 +. /scripts/common.sh + +wait_topic fb-sink + +exec /usr/local/bin/fluent-bit -c /etc/kafka.conf diff --git a/fluent-bit/examples/kafka_filter/scripts/kafka-consume.sh b/fluent-bit/examples/kafka_filter/scripts/kafka-consume.sh new file mode 100755 index 00000000..729c4876 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/scripts/kafka-consume.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ue + +# shellcheck disable=SC1091 +. /scripts/common.sh + +wait_topic fb-sink + +kafka-console-consumer --topic fb-sink --bootstrap-server \ + "$KAFKA_HOST:$KAFKA_PORT" diff --git a/fluent-bit/examples/kafka_filter/scripts/kafka-produce.sh b/fluent-bit/examples/kafka_filter/scripts/kafka-produce.sh new file mode 100755 index 00000000..eea99ca2 --- /dev/null +++ b/fluent-bit/examples/kafka_filter/scripts/kafka-produce.sh @@ -0,0 +1,13 @@ +#!/bin/bash -ue + +# shellcheck disable=SC1091 +. /scripts/common.sh + +wait_topic fb-sink + +for i in $(seq 1 100); do + sleep 1 + echo "{ \"name\": \"object-$i\" }" | \ + kafka-console-producer --topic fb-source \ + --broker-list "$KAFKA_HOST:$KAFKA_PORT" +done diff --git a/fluent-bit/examples/out_lib/CMakeLists.txt b/fluent-bit/examples/out_lib/CMakeLists.txt new file mode 100644 index 00000000..db04ff1c --- /dev/null +++ b/fluent-bit/examples/out_lib/CMakeLists.txt @@ -0,0 +1,9 @@ +set(src + ${src} + out_lib.c + ) + +find_package (Threads) +add_executable(out_lib ${src}) +target_link_libraries(out_lib fluent-bit-shared) +target_link_libraries(out_lib ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fluent-bit/examples/out_lib/README.md b/fluent-bit/examples/out_lib/README.md new file mode 100644 index 00000000..f9fc5a4e --- /dev/null +++ b/fluent-bit/examples/out_lib/README.md @@ -0,0 +1,6 @@ +# Fluent Bit / Hello World + +This tool uses the Fluent Bit library version to flush data to one of the +registered outputs. + +When using Fluent Bit as a library means the caller becomes the input. diff --git a/fluent-bit/examples/out_lib/out_lib.c b/fluent-bit/examples/out_lib/out_lib.c new file mode 100644 index 00000000..ada207b0 --- /dev/null +++ b/fluent-bit/examples/out_lib/out_lib.c @@ -0,0 +1,94 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit Demo + * =============== + * Copyright (C) 2015-2017 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit.h> +#include <msgpack.h> + +int my_stdout_json(void *record, size_t size, void *data) +{ + printf("[%s]",__FUNCTION__); + printf("%s",(char*)record); + printf("\n"); + + flb_lib_free(record); + return 0; +} + +int my_stdout_msgpack(void* data, size_t size) +{ + printf("[%s]",__FUNCTION__); + msgpack_object_print(stdout, *(msgpack_object*)data); + printf("\n"); + + flb_lib_free(data); + return 0; +} + +int main() +{ + int i; + int n; + char tmp[256]; + flb_ctx_t *ctx; + struct flb_lib_out_cb callback; + int in_ffd; + int out_ffd; + + /* Initialize library */ + ctx = flb_create(); + if (!ctx) { + exit(EXIT_FAILURE); + } + + in_ffd = flb_input(ctx, "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Register my callback function */ + + /* JSON format */ + callback.cb = my_stdout_json; + callback.data = NULL; + + out_ffd = flb_output(ctx, "lib", &callback); + flb_output_set(ctx, out_ffd, "match", "test", "format", "json", NULL); + + /* Msgpack format */ + /* + out_ffd = flb_output(ctx, "lib", my_stdout_msgpack); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + */ + + /* Start the background worker */ + flb_start(ctx); + + /* Push some data */ + for (i = 0; i < 100; i++) { + n = snprintf(tmp, sizeof(tmp) - 1, + "[%f, {\"key\": \"val %i\"}]", + flb_time_now(), i); + flb_lib_push(ctx, in_ffd, tmp, n); + } + + flb_stop(ctx); + + /* Release Resources */ + flb_destroy(ctx); + + return 0; +} diff --git a/fluent-bit/examples/perf_test/.env b/fluent-bit/examples/perf_test/.env new file mode 100644 index 00000000..c24399e9 --- /dev/null +++ b/fluent-bit/examples/perf_test/.env @@ -0,0 +1,5 @@ +FB_BASELINE_IMAGE=fluent/fluent-bit:latest +LOG_PREFIX=test +LOG_COUNT=100 +LOG_RATE=20 +LOG_SIZE=1000 diff --git a/fluent-bit/examples/perf_test/docker-compose.yml b/fluent-bit/examples/perf_test/docker-compose.yml new file mode 100644 index 00000000..9f67440a --- /dev/null +++ b/fluent-bit/examples/perf_test/docker-compose.yml @@ -0,0 +1,38 @@ +version: "3.4" + +services: + + fb-baseline: + image: ${FB_BASELINE_IMAGE} + volumes: + - log-volume:/logs:ro + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro + + fb-delta: + build: + context: ../.. + dockerfile: dockerfiles/Dockerfile + target: production + volumes: + - log-volume:/logs:ro + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro + ports: + - "2020:2020" + + data-generator: + image: fluentbitdev/fluent-bit-ci:benchmark + command: bash /scripts/entrypoint.sh + volumes: + - log-volume:/logs:rw + - ./scripts:/scripts:ro + environment: + LOG_PREFIX: ${LOG_PREFIX} + LOG_COUNT: ${LOG_COUNT} + LOG_SIZE: ${LOG_SIZE} + # For multiline set this to >0 + LINE_COUNT: ${LINE_COUNT} + # For multiline set this to the sleep period between log entries rather than the rate + LOG_RATE: ${LOG_RATE} + +volumes: + log-volume: diff --git a/fluent-bit/examples/perf_test/fluent-bit.conf b/fluent-bit/examples/perf_test/fluent-bit.conf new file mode 100644 index 00000000..3defa36e --- /dev/null +++ b/fluent-bit/examples/perf_test/fluent-bit.conf @@ -0,0 +1,27 @@ +[SERVICE] + Daemon Off + # Log_Level debug + HTTP_Server On + # HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + flush 1 + storage.metrics on + storage.path /data/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 1G + storage.max_chunks_up 128 + +[INPUT] + Name dummy + fixed_timestamp on + +[INPUT] + Name tail + Path /logs/* + # Intended for use with multiline data generator + multiline.parser java + +[OUTPUT] + Name null + Match * diff --git a/fluent-bit/examples/perf_test/scripts/entrypoint.sh b/fluent-bit/examples/perf_test/scripts/entrypoint.sh new file mode 100755 index 00000000..4b53d4c7 --- /dev/null +++ b/fluent-bit/examples/perf_test/scripts/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -eu + +LOG_DIR=${LOG_DIR:-/logs} +LOG_PREFIX=${LOG_PREFIX:-multi} +LOG_COUNT=${LOG_COUNT:-100} +LOG_RATE=${LOG_RATE:-20} +LOG_SIZE=${LOG_SIZE:-1000} +LINE_COUNT=${LINE_COUNT:-0} + +rm -vfr "${LOG_DIR:?}/$LOG_PREFIX*" + +for i in $(seq "$LOG_COUNT") +do + export OUTPUT_LOGFILE="$LOG_DIR/$LOG_PREFIX-$i.log" + if [[ "$LINE_COUNT" -gt 0 ]]; then + /scripts/multi-line-log-generator.sh & + else + echo "Creating $OUTPUT_LOGFILE" + # Far too much debug + /run_log_generator.py --log-size-in-bytes "$LOG_SIZE" --log-rate "$LOG_RATE" --log-agent-input-type tail --tail-file-path "$OUTPUT_LOGFILE" &> /dev/null & + fi +done + +wait diff --git a/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh b/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh new file mode 100755 index 00000000..1e6236d4 --- /dev/null +++ b/fluent-bit/examples/perf_test/scripts/multi-line-log-generator.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eu + + +OUTPUT_LOGFILE=${OUTPUT_LOGFILE:-/logs/test.log} +rm -fv "$OUTPUT_LOGFILE" + +LOG_RATE=${LOG_RATE:-0.2} +LINE_COUNT=${LINE_COUNT:-100} + +echo "Sleep for $LOG_RATE and create $OUTPUT_LOGFILE with $LINE_COUNT+1 lines per entry" + +while true; do + cat >> "$OUTPUT_LOGFILE" << EOF +Exception in thread "main" java.lang.RuntimeException: A test exception +EOF + for _ in $(seq "$LINE_COUNT"); do +cat >> "$OUTPUT_LOGFILE" << EOF + at com.stackify.stacktrace.StackTraceExample.methodB(StackTraceExample.java:13) +EOF + done + sleep "$LOG_RATE" +done diff --git a/fluent-bit/examples/tracing/docker-compose.yml b/fluent-bit/examples/tracing/docker-compose.yml new file mode 100644 index 00000000..94d00634 --- /dev/null +++ b/fluent-bit/examples/tracing/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" + +services: + fluent-bit: + build: + context: ../.. + dockerfile: dockerfiles/Dockerfile + args: + - FLB_CHUNK_TRACE=On + ports: + - "2020:2020" + volumes: + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro + command: + - /fluent-bit/bin/fluent-bit + - -Z + - -c + - /fluent-bit/etc/fluent-bit.conf + + enable-tracing: + image: ubuntu:20.04 + command: /enable-tracing.sh + volumes: + - ./enable-tracing.sh:/enable-tracing.sh:ro + environment: + REMOTE_HOST: fluent-bit diff --git a/fluent-bit/examples/tracing/enable-tracing.sh b/fluent-bit/examples/tracing/enable-tracing.sh new file mode 100755 index 00000000..5bff2b90 --- /dev/null +++ b/fluent-bit/examples/tracing/enable-tracing.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -eu + +REMOTE_HOST=${REMOTE_HOST:-127.0.0.1} + +if command -v http &> /dev/null ; then + http -v "$REMOTE_HOST":2020/api/v1/trace/dummy.0 output=stdout prefix=trace. params:='{"format":"json"}' +elif command -v curl &> /dev/null ; then + curl --header 'Content-Type: application/json' --data '{"output": "stdout", "params": { "format": "json" }, "prefix": "trace."}' "$REMOTE_HOST":2020/api/v1/trace/dummy.0 +else + echo "No curl or httpie installed" + if command -v apt-get &> /dev/null ; then + apt-get -qq update + apt-get -qq install -y curl + elif command -v yum &> /dev/null ; then + yum install -y curl + else + exit 1 + fi + curl --header 'Content-Type: application/json' --data '{"output": "stdout", "params": { "format": "json" }, "prefix": "trace."}' "$REMOTE_HOST":2020/api/v1/trace/dummy.0 +fi diff --git a/fluent-bit/examples/tracing/fluent-bit.conf b/fluent-bit/examples/tracing/fluent-bit.conf new file mode 100644 index 00000000..0d83db70 --- /dev/null +++ b/fluent-bit/examples/tracing/fluent-bit.conf @@ -0,0 +1,14 @@ +[SERVICE] + Http_Server On + +[INPUT] + Name dummy + +[FILTER] + Name record_modifier + Match * + Record foo bar + +[OUTPUT] + Name stdout + Match * diff --git a/fluent-bit/examples/wasi_serde_json/Cargo.lock b/fluent-bit/examples/wasi_serde_json/Cargo.lock new file mode 100644 index 00000000..e2006691 --- /dev/null +++ b/fluent-bit/examples/wasi_serde_json/Cargo.lock @@ -0,0 +1,123 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" + +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi_serde_json" +version = "0.1.0" +dependencies = [ + "chrono", + "serde_json", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/fluent-bit/examples/wasi_serde_json/Cargo.toml b/fluent-bit/examples/wasi_serde_json/Cargo.toml new file mode 100644 index 00000000..c9778b46 --- /dev/null +++ b/fluent-bit/examples/wasi_serde_json/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "wasi_serde_json" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde_json = "1.0" +chrono = "0.4"
\ No newline at end of file diff --git a/fluent-bit/examples/wasi_serde_json/README.md b/fluent-bit/examples/wasi_serde_json/README.md new file mode 100644 index 00000000..9df44189 --- /dev/null +++ b/fluent-bit/examples/wasi_serde_json/README.md @@ -0,0 +1,65 @@ +# Fluent Bit / wasi_serde_json + +This source source tree provides an example of WASM program with WASI mode. + +## Prerequisites + +* Rust + * rustc 1.61.0 (fe5b13d68 2022-05-18) +* [rustup](https://rustup.rs/) (For preparing rust compiler and toolchains) + +## How to build + +Add `wasm32-wasi` target for Rust toolchain: + +```console +$ rustup target add wasm32-wasi +``` + +Then, execute _cargo build_ as follows: + +```console +$ cargo build --target wasm32-wasi --release +``` + +Finally, under target/wasm32-wasi/release directory, `*.wasm` file will be created: + +```console +$ ls target/wasm32-wasi/release/*.wasm +target/wasm32-wasi/release/wasi_serde_json.wasm +``` + +## How to confirm WASI integration + +Create parsers.conf as follows: + +```ini +[PARSER] + Name wasi + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L %z +``` + +And Create fluent-bit configuration file as follows: + +```ini +[SERVICE] + Flush 1 + Daemon Off + Parsers_File parsers.conf + Log_Level info + HTTP_Server Off + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name exec_wasi + Tag exec.wasi.local + WASI_Path /path/to/wasi_serde_json.wasm + Parser wasi + +[OUTPUT] + Name stdout + Match * +``` diff --git a/fluent-bit/examples/wasi_serde_json/src/main.rs b/fluent-bit/examples/wasi_serde_json/src/main.rs new file mode 100644 index 00000000..29ee25f4 --- /dev/null +++ b/fluent-bit/examples/wasi_serde_json/src/main.rs @@ -0,0 +1,32 @@ +// Import rust's io and filesystem module +use std::io::prelude::*; +use std::fs; +// Import pure and fast JSON library written in Rust +use serde_json::json; +// Import chrono library to handle time related operation conveniently +use chrono::Utc; + +// Entry point to our WASI applications +fn main() { + // Note that fractional second must be handled by `.%9f` not `%L` like as fluent-bit. + let time = Utc::now().format("%Y-%m-%dT%H:%M:%S.%9f %z").to_string(); + // The type of `john` is `serde_json::Value` + let john = json!({ + "name": "John Doe", + "age": 43, + "phones": [ + "+44 1234567", + "+44 2345678" + ], + "time": format!("{}", time), + }); + // Print out serialized JSON data for john + // This will handle writing to stdout for us using the WASI APIs (e.g fd_write) + println!("{}", john.to_string()); + + // Create a file (Testing for exposed wasi API) + let mut file = fs::File::create("helloworld.txt").expect("Unable to create file"); + + // Write the text to the file we created (Testing for exposed wasi API) + write!(file, "Hello world!\n").unwrap(); +}
\ No newline at end of file diff --git a/fluent-bit/examples/windows/docker/README.md b/fluent-bit/examples/windows/docker/README.md new file mode 100644 index 00000000..7efbb196 --- /dev/null +++ b/fluent-bit/examples/windows/docker/README.md @@ -0,0 +1,28 @@ +# Fluent Bit running in Windows Container
+
+You can test the Fluent Bit image on a Windows Container host as follows:
+
+```
+PS> $SAMPLE_DIR="<REPO_CHECKOUT_FOLDER>\examples\windows"
+PS> docker run -v $SAMPLE_DIR\docker\conf:c:\config -v $SAMPLE_DIR\docker\logs:c:\logs -v $SAMPLE_DIR\docker\state:c:\state fluent/fluent-bit:1.3.7-nanoserver
+```
+
+You should see something like the following output:
+
+```
+Fluent Bit v1.3.7
+Copyright (C) Treasure Data
+
+[2020/02/16 16:02:02] [ info] [storage] initializing...
+[2020/02/16 16:02:02] [ info] [storage] in-memory
+[2020/02/16 16:02:02] [ info] [storage] normal synchronization mode, checksum disabled, max_chunks_up=128
+[2020/02/16 16:02:02] [ info] [engine] started (pid=1256)
+[2020/02/16 16:02:02] [ info] [sp] stream processor started
+{"date":"2020-02-16T06:02:02.951104Z","log":"{\"date\":\"2019-08-12T05:28:39.075296Z\",\"log\":\"2019-08-12 05:28:29 10.240.0.77 GET /bundles/MsAjaxJs v=c42ygB2U07n37m_Sfa8ZbLGVu4Rr2gsBo7MvUEnJeZ81 80 - 10.240.0.35 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 65\"}"}
+{"date":"2020-02-16T06:02:02.951136Z","log":"{\"date\":\"2019-08-12T05:28:39.075296Z\",\"log\":\"2019-08-12 05:28:29 10.240.0.77 GET /Scripts/respond.js - 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 40\"}"}
+{"date":"2020-02-16T06:02:02.951137Z","log":"{\"date\":\"2019-08-12T05:28:39.075296Z\",\"log\":\"2019-08-12 05:28:29 10.240.0.77 GET /Content/Site.css - 80 - 10.240.0.35 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 70\"}"}
+{"date":"2020-02-16T06:02:02.951137Z","log":"{\"date\":\"2019-08-12T05:28:39.075297Z\",\"log\":\"2019-08-12 05:28:29 10.240.0.77 GET /bundles/WebFormsJs v=AAyiAYwMfvmwjNSBfIMrBAqfU5exDukMVhrRuZ-PDU01 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 "}
+{"date":"2020-02-16T06:02:02.951138Z","log":"0 47\"}"}
+{"date":"2020-02-16T06:02:02.951138Z","log":"{\"date\":\"2019-08-12T05:28:39.075297Z\",\"log\":\"2019-08-12 05:28:29 10.240.0.77 GET /favicon.ico - 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 28\"}"}
+{"date":"2020-02-16T06:02:02.951138Z","log":"{\"date\":\"2019-08-12T05:31:37.941523Z\",\"log\":\"2019-08-12 05:30:57 10.240.0.77 GET / - 80 - 10.240.0.4 - - 200 0 64 769\"}"}
+```
\ No newline at end of file diff --git a/fluent-bit/examples/windows/docker/conf/fluent-bit.conf b/fluent-bit/examples/windows/docker/conf/fluent-bit.conf new file mode 100644 index 00000000..1fca9d29 --- /dev/null +++ b/fluent-bit/examples/windows/docker/conf/fluent-bit.conf @@ -0,0 +1,18 @@ +[SERVICE]
+ Flush 5
+ Daemon off
+ Log_Level info
+
+[INPUT]
+ Name tail
+ Path c:\\logs\\*.log
+ Refresh_Interval 60
+ Ignore_Older 10s
+ Rotate_Wait 5
+ DB c:\\state\\logs.db
+
+[OUTPUT]
+ Name stdout
+ Match *
+ Format json_lines
+ json_date_format iso8601
\ No newline at end of file diff --git a/fluent-bit/examples/windows/docker/logs/application.log b/fluent-bit/examples/windows/docker/logs/application.log new file mode 100644 index 00000000..9ad9ee96 --- /dev/null +++ b/fluent-bit/examples/windows/docker/logs/application.log @@ -0,0 +1,7 @@ +{"date":"2019-08-12T05:28:39.075296Z","log":"2019-08-12 05:28:29 10.240.0.77 GET /bundles/MsAjaxJs v=c42ygB2U07n37m_Sfa8ZbLGVu4Rr2gsBo7MvUEnJeZ81 80 - 10.240.0.35 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 65"}
+{"date":"2019-08-12T05:28:39.075296Z","log":"2019-08-12 05:28:29 10.240.0.77 GET /Scripts/respond.js - 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 40"}
+{"date":"2019-08-12T05:28:39.075296Z","log":"2019-08-12 05:28:29 10.240.0.77 GET /Content/Site.css - 80 - 10.240.0.35 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 70"}
+{"date":"2019-08-12T05:28:39.075297Z","log":"2019-08-12 05:28:29 10.240.0.77 GET /bundles/WebFormsJs v=AAyiAYwMfvmwjNSBfIMrBAqfU5exDukMVhrRuZ-PDU01 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0
+0 47"}
+{"date":"2019-08-12T05:28:39.075297Z","log":"2019-08-12 05:28:29 10.240.0.77 GET /favicon.ico - 80 - 10.240.0.4 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/76.0.3809.100+Safari/537.36 http://52.237.212.148/Contact 200 0 0 28"}
+{"date":"2019-08-12T05:31:37.941523Z","log":"2019-08-12 05:30:57 10.240.0.77 GET / - 80 - 10.240.0.4 - - 200 0 64 769"}
\ No newline at end of file diff --git a/fluent-bit/examples/windows/kubernetes/README.md b/fluent-bit/examples/windows/kubernetes/README.md new file mode 100644 index 00000000..f750a5d1 --- /dev/null +++ b/fluent-bit/examples/windows/kubernetes/README.md @@ -0,0 +1,27 @@ +# Fluent Bit running as a sidecar on Kubernetes Windows node
+
+You can test the Fluent Bit image as a sidecar on a Kubernetes Windows node as follows:
+
+```
+kubectl create namespace fluentbit
+kubectl apply -n fluentbit -f $SAMPLE_DIR\kubernetes\configmap.yaml
+kubectl apply -n fluentbit -f $SAMPLE_DIR\kubernetes\deployment.yaml
+```
+
+Ensure that the resources have been correctly deployed.
+
+```
+kubectl get pod -n fluentbit
+NAME READY STATUS RESTARTS AGE
+logging-fluentbit-sidecar-6ff8c84494-zt4ft 2/2 Running 0 31m
+
+kubectl get svc -n fluentbit
+NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+logging-fluentbit-sidecar LoadBalancer 10.0.27.172 52.237.212.148 80:30172/TCP 126m
+```
+
+If you tail the logs on the Fluent Bit sidecar, and then explore the website running in the external ip and port 80, you should see generated logs ...
+
+```
+kubectl logs logging-fluentbit-sidecar-6ff8c84494-zt4ft -n fluentbit -c fluentbit-logger -f
+```
\ No newline at end of file diff --git a/fluent-bit/examples/windows/kubernetes/configmap.yaml b/fluent-bit/examples/windows/kubernetes/configmap.yaml new file mode 100644 index 00000000..18ee0b99 --- /dev/null +++ b/fluent-bit/examples/windows/kubernetes/configmap.yaml @@ -0,0 +1,24 @@ +apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: fluentbit-config
+data:
+ fluent-bit.conf: |-
+ [SERVICE]
+ Flush 5
+ Daemon off
+ Log_Level info
+
+ [INPUT]
+ Name tail
+ Path c:\\logs\\*.log
+ Refresh_Interval 60
+ Ignore_Older 10s
+ Rotate_Wait 5
+ DB c:\\state\\logs.db
+
+ [OUTPUT]
+ Name stdout
+ Match *
+ Format json_lines
+ json_date_format iso8601
diff --git a/fluent-bit/examples/windows/kubernetes/deployment.yaml b/fluent-bit/examples/windows/kubernetes/deployment.yaml new file mode 100644 index 00000000..7bbf2996 --- /dev/null +++ b/fluent-bit/examples/windows/kubernetes/deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: v1
+kind: Service
+metadata:
+ name: logging-fluentbit-sidecar
+ labels:
+ app: logging-fluentbit-sidecar
+spec:
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ selector:
+ app: website
+ type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: logging-fluentbit-sidecar
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: website
+ template:
+ metadata:
+ labels:
+ app: website
+ spec:
+ containers:
+ - name: aspnet-web
+ image: mcr.microsoft.com/dotnet/framework/samples:aspnetapp
+ volumeMounts:
+ - mountPath: c:\inetpub\logs\LogFiles\W3SVC1\
+ name: log-volume
+ - name: fluentbit-logger
+ image: fluent/fluent-bit:1.3.7-nanoserver
+ imagePullPolicy: Always
+ volumeMounts:
+ - mountPath: c:\config
+ name: config
+ - mountPath: c:\logs
+ name: log-volume
+ - mountPath: c:\state
+ name: log-state
+ volumes:
+ - name: config
+ configMap:
+ name: fluentbit-config
+ - name: log-volume
+ emptyDir: {}
+ - name: log-state
+ emptyDir: {}
+ nodeSelector:
+ beta.kubernetes.io/os: windows
\ No newline at end of file |